Repository: incubator-ariatosca Updated Branches: refs/heads/ARIA-21-extract-some-parser-code 3e7f1e15a -> 3895f8ca6 (forced update)
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/3895f8ca/aria/utils/caching.py ---------------------------------------------------------------------- diff --git a/aria/utils/caching.py b/aria/utils/caching.py new file mode 100644 index 0000000..613ade6 --- /dev/null +++ b/aria/utils/caching.py @@ -0,0 +1,133 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import absolute_import # so we can import standard 'collections' and 'threading' + +from threading import Lock +from functools import partial + +from .collections import OrderedDict + + +class cachedmethod(object): # pylint: disable=invalid-name + """ + Decorator for caching method return values. + + The implementation is thread-safe. + + Supports :code:`cache_info` to be compatible with Python 3's :code:`functools.lru_cache`. + Note that the statistics are combined for all instances of the class. + + Won't use the cache if not called when bound to an object, allowing you to override the cache. + + Adapted from `this solution + <http://code.activestate.com/recipes/577452-a-memoize-decorator-for-instance-methods/>`__. + """ + + ENABLED = True + + def __init__(self, func): + self.func = func + self.hits = 0 + self.misses = 0 + self.lock = Lock() + + def cache_info(self): + with self.lock: + return (self.hits, self.misses, None, self.misses) + + def reset_cache_info(self): + with self.lock: + self.hits = 0 + self.misses = 0 + + def __get__(self, instance, owner): + if instance is None: + # Don't use cache if not bound to an object + # Note: This is also a way for callers to override the cache + return self.func + return partial(self, instance) + + def __call__(self, *args, **kwargs): + if not self.ENABLED: + return self.func(*args, **kwargs) + + instance = args[0] + cache = instance.get_method_cache() + + key = (self.func, args[1:], frozenset(kwargs.items())) + + try: + with self.lock: + return_value = cache[key] + self.hits += 1 + except KeyError: + return_value = self.func(*args, **kwargs) + with self.lock: + cache[key] = return_value + self.misses += 1 + # Another thread may override our cache entry here, so we need to read + # it again to make sure all threads use the same return value + return_value = cache.get(key, return_value) + + return return_value + + +class HasCachedMethods(object): + """ + Provides convenience methods for working with :class:`cachedmethod`. + """ + + def __init__(self, method_cache=None): + self._method_cache = method_cache or {} + + def get_method_cache(self): + return self._method_cache + + @property + def _method_cache_info(self): + """ + The cache infos of all cached methods. + + :rtype: dict of str, 4-tuple + """ + + cached_info = OrderedDict() + for k, v in self.__class__.__dict__.iteritems(): + if isinstance(v, property): + # The property getter might be cached + v = v.fget + if hasattr(v, 'cache_info'): + cached_info[k] = v.cache_info() + return cached_info + + def _reset_method_cache(self): + """ + Resets the caches of all cached methods. + """ + + if hasattr(self, '_method_cache'): + self._method_cache = {} + + # Note: Another thread may already be storing entries in the cache here. + # But it's not a big deal! It only means that our cache_info isn't + # guaranteed to be accurate. + + for entry in self.__class__.__dict__.itervalues(): + if isinstance(entry, property): + # The property getter might be cached + entry = entry.fget + if hasattr(entry, 'reset_cache_info'): + entry.reset_cache_info() http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/3895f8ca/aria/utils/collections.py ---------------------------------------------------------------------- diff --git a/aria/utils/collections.py b/aria/utils/collections.py new file mode 100644 index 0000000..03feabd --- /dev/null +++ b/aria/utils/collections.py @@ -0,0 +1,291 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import absolute_import # so we can import standard 'collections' + +from copy import deepcopy +try: + from collections import OrderedDict +except ImportError: + from ordereddict import OrderedDict + + +def cls_name(cls): + module = str(cls.__module__) + name = str(cls.__name__) + return name if module == '__builtin__' else '%s.%s' % (module, name) + + +class FrozenList(list): + """ + An immutable list. + + After initialization it will raise :class:`TypeError` exceptions if modification + is attempted. + + Note that objects stored in the list may not be immutable. + """ + def __init__(self, *args, **kwargs): + self.locked = False + super(FrozenList, self).__init__(*args, **kwargs) + self.locked = True + + def __setitem__(self, index, value): + if self.locked: + raise TypeError('frozen list') + return super(FrozenList, self).__setitem__(index, value) + + def __delitem__(self, index): + if self.locked: + raise TypeError('frozen list') + return super(FrozenList, self).__delitem__(index) + + def __iadd__(self, values): + if self.locked: + raise TypeError('frozen list') + return super(FrozenList, self).__iadd__(values) + + def __deepcopy__(self, memo): + res = [deepcopy(v, memo) for v in self] + return FrozenList(res) + + def append(self, value): + if self.locked: + raise TypeError('frozen list') + return super(FrozenList, self).append(value) + + def extend(self, values): + if self.locked: + raise TypeError('frozen list') + return super(FrozenList, self).append(values) + + def insert(self, index, value): + if self.locked: + raise TypeError('frozen list') + return super(FrozenList, self).insert(index, value) + +EMPTY_READ_ONLY_LIST = FrozenList() + +class FrozenDict(OrderedDict): + """ + An immutable ordered dict. + + After initialization it will raise :class:`TypeError` exceptions if modification + is attempted. + + Note that objects stored in the dict may not be immutable. + """ + + def __init__(self, *args, **kwargs): + self.locked = False + super(FrozenDict, self).__init__(*args, **kwargs) + self.locked = True + + def __setitem__(self, key, value, **_): + if self.locked: + raise TypeError('frozen dict') + return super(FrozenDict, self).__setitem__(key, value) + + def __delitem__(self, key, **_): + if self.locked: + raise TypeError('frozen dict') + return super(FrozenDict, self).__delitem__(key) + + def __deepcopy__(self, memo): + res = [(deepcopy(k, memo), deepcopy(v, memo)) for k, v in self.iteritems()] + return FrozenDict(res) + +EMPTY_READ_ONLY_DICT = FrozenDict() + +class StrictList(list): + """ + A list that raises :class:`TypeError` exceptions when objects of the wrong type are inserted. + """ + + def __init__(self, + items=None, + value_class=None, + wrapper_function=None, + unwrapper_function=None): + super(StrictList, self).__init__() + if isinstance(items, StrictList): + self.value_class = items.value_class + self.wrapper_function = items.wrapper_function + self.unwrapper_function = items.unwrapper_function + self.value_class = value_class + self.wrapper_function = wrapper_function + self.unwrapper_function = unwrapper_function + if items: + for item in items: + self.append(item) + + def _wrap(self, value): + if (self.value_class is not None) and (not isinstance(value, self.value_class)): + raise TypeError('value must be a "%s": %s' % (cls_name(self.value_class), repr(value))) + if self.wrapper_function is not None: + value = self.wrapper_function(value) + return value + + def _unwrap(self, value): + if self.unwrapper_function is not None: + value = self.unwrapper_function(value) + return value + + def __getitem__(self, index): + value = super(StrictList, self).__getitem__(index) + value = self._unwrap(value) + return value + + def __setitem__(self, index, value): + value = self._wrap(value) + return super(StrictList, self).__setitem__(index, value) + + def __iadd__(self, values): + values = [self._wrap(v) for v in values] + return super(StrictList, self).__iadd__(values) + + def append(self, value): + value = self._wrap(value) + return super(StrictList, self).append(value) + + def extend(self, values): + values = [self._wrap(v) for v in values] + return super(StrictList, self).extend(values) + + def insert(self, index, value): + value = self._wrap(value) + return super(StrictList, self).insert(index, value) + +class StrictDict(OrderedDict): + """ + An ordered dict that raises :class:`TypeError` exceptions + when keys or values of the wrong type are used. + """ + + def __init__(self, + items=None, + key_class=None, + value_class=None, + wrapper_function=None, + unwrapper_function=None): + super(StrictDict, self).__init__() + if isinstance(items, StrictDict): + self.key_class = items.key_class + self.value_class = items.value_class + self.wrapper_function = items.wrapper_function + self.unwrapper_function = items.unwrapper_function + self.key_class = key_class + self.value_class = value_class + self.wrapper_function = wrapper_function + self.unwrapper_function = unwrapper_function + if items: + for k, v in items: + self[k] = v + + def __getitem__(self, key): + if (self.key_class is not None) and (not isinstance(key, self.key_class)): + raise TypeError('key must be a "%s": %s' % (cls_name(self.key_class), repr(key))) + value = super(StrictDict, self).__getitem__(key) + if self.unwrapper_function is not None: + value = self.unwrapper_function(value) + return value + + def __setitem__(self, key, value, **_): + if (self.key_class is not None) and (not isinstance(key, self.key_class)): + raise TypeError('key must be a "%s": %s' % (cls_name(self.key_class), repr(key))) + if (self.value_class is not None) and (not isinstance(value, self.value_class)): + raise TypeError('value must be a "%s": %s' % (cls_name(self.value_class), repr(value))) + if self.wrapper_function is not None: + value = self.wrapper_function(value) + return super(StrictDict, self).__setitem__(key, value) + +def merge(dict_a, dict_b, path=None, strict=False): + """ + Merges dicts, recursively. + """ + + # TODO: a.add_yaml_merge(b), see https://bitbucket.org/ruamel/yaml/src/ + # TODO: 86622a1408e0f171a12e140d53c4ffac4b6caaa3/comments.py?fileviewer=file-view-default + + path = path or [] + for key, value_b in dict_b.iteritems(): + if key in dict_a: + value_a = dict_a[key] + if isinstance(value_a, dict) and isinstance(value_b, dict): + merge(value_a, value_b, path + [str(key)], strict) + elif value_a != value_b: + if strict: + raise ValueError('dict merge conflict at %s' % '.'.join(path + [str(key)])) + else: + dict_a[key] = value_b + else: + dict_a[key] = value_b + return dict_a + +def is_removable(_container, _key, v): + return (v is None) or ((isinstance(v, dict) or isinstance(v, list)) and (len(v) == 0)) + +def prune(value, is_removable_function=is_removable): + """ + Deletes :code:`None` and empty lists and dicts, recursively. + """ + + if isinstance(value, list): + for i, v in enumerate(value): + if is_removable_function(value, i, v): + del value[i] + else: + prune(v, is_removable_function) + elif isinstance(value, dict): + for k, v in value.iteritems(): + if is_removable_function(value, k, v): + del value[k] + else: + prune(v, is_removable_function) + + return value + + +# TODO (left for tal): Move following two methods to some place parser specific +def deepcopy_with_locators(value): + """ + Like :code:`deepcopy`, but also copies over locators. + """ + + res = deepcopy(value) + copy_locators(res, value) + return res + + +def copy_locators(target, source): + """ + Copies over :code:`_locator` for all elements, recursively. + + Assumes that target and source have exactly the same list/dict structure. + """ + + locator = getattr(source, '_locator', None) + if locator is not None: + try: + setattr(target, '_locator', locator) + except AttributeError: + pass + + if isinstance(target, list) and isinstance(source, list): + for i, _ in enumerate(target): + copy_locators(target[i], source[i]) + elif isinstance(target, dict) and isinstance(source, dict): + for k, v in target.iteritems(): + copy_locators(v, source[k]) http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/3895f8ca/aria/utils/console.py ---------------------------------------------------------------------- diff --git a/aria/utils/console.py b/aria/utils/console.py new file mode 100644 index 0000000..55d2529 --- /dev/null +++ b/aria/utils/console.py @@ -0,0 +1,64 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from clint.textui.core import STDOUT +from clint.textui import puts as _puts +from clint.textui.colored import ColoredString as _ColoredString +from clint.textui import indent # pylint: disable=unused-import + +from .formatting import safe_str + + +class ColoredString(_ColoredString): + def __init__(self, color, str_, always_color=False, bold=False): + super(ColoredString, self).__init__(color, safe_str(str_), always_color, bold) + + +def puts(string='', newline=True, stream=STDOUT): + _puts(safe_str(string), newline, stream) + + +class Colored(object): + @staticmethod + def black(string, always=False, bold=False): + return ColoredString('BLACK', string, always_color=always, bold=bold) + + @staticmethod + def red(string, always=False, bold=False): + return ColoredString('RED', string, always_color=always, bold=bold) + + @staticmethod + def green(string, always=False, bold=False): + return ColoredString('GREEN', string, always_color=always, bold=bold) + + @staticmethod + def yellow(string, always=False, bold=False): + return ColoredString('YELLOW', string, always_color=always, bold=bold) + + @staticmethod + def blue(string, always=False, bold=False): + return ColoredString('BLUE', string, always_color=always, bold=bold) + + @staticmethod + def magenta(string, always=False, bold=False): + return ColoredString('MAGENTA', string, always_color=always, bold=bold) + + @staticmethod + def cyan(string, always=False, bold=False): + return ColoredString('CYAN', string, always_color=always, bold=bold) + + @staticmethod + def white(string, always=False, bold=False): + return ColoredString('WHITE', string, always_color=always, bold=bold) http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/3895f8ca/aria/utils/exceptions.py ---------------------------------------------------------------------- diff --git a/aria/utils/exceptions.py b/aria/utils/exceptions.py new file mode 100644 index 0000000..0370bb3 --- /dev/null +++ b/aria/utils/exceptions.py @@ -0,0 +1,64 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys +import linecache + +from clint.textui import indent +from .console import (puts, Colored) + + +def print_exception(e, full=True, cause=False, traceback=None): + """ + Prints the exception with nice colors and such. + """ + def format_heading(e): + return '%s%s: %s' % (Colored.red('Caused by ') if cause else '', Colored.red( + e.__class__.__name__, bold=True), Colored.red(e)) + + puts(format_heading(e)) + if full: + if cause: + if traceback: + print_traceback(traceback) + else: + print_traceback() + if hasattr(e, 'cause') and e.cause: + traceback = e.cause_traceback if hasattr(e, 'cause_traceback') else None + print_exception(e.cause, full=full, cause=True, traceback=traceback) + +def print_traceback(traceback=None): + """ + Prints the traceback with nice colors and such. + """ + + if traceback is None: + _, _, traceback = sys.exc_info() + while traceback is not None: + frame = traceback.tb_frame + lineno = traceback.tb_lineno + code = frame.f_code + filename = code.co_filename + name = code.co_name + with indent(2): + puts('File "%s", line %s, in %s' % (Colored.blue(filename), + Colored.cyan(lineno), + Colored.cyan(name))) + linecache.checkcache(filename) + line = linecache.getline(filename, lineno, frame.f_globals) + if line: + with indent(2): + puts(Colored.black(line.strip())) + traceback = traceback.tb_next http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/3895f8ca/aria/utils/formatting.py ---------------------------------------------------------------------- diff --git a/aria/utils/formatting.py b/aria/utils/formatting.py new file mode 100644 index 0000000..3725bc7 --- /dev/null +++ b/aria/utils/formatting.py @@ -0,0 +1,203 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import absolute_import # so we can import standard 'collections' + +import json +from types import MethodType +from ruamel import yaml # @UnresolvedImport + +from .collections import FrozenList, FrozenDict, StrictList, StrictDict, OrderedDict + +# Add our types to ruamel.yaml (for round trips) +yaml.representer.RoundTripRepresenter.add_representer( + FrozenList, yaml.representer.RoundTripRepresenter.represent_list) +yaml.representer.RoundTripRepresenter.add_representer( + FrozenDict, yaml.representer.RoundTripRepresenter.represent_dict) +yaml.representer.RoundTripRepresenter.add_representer( + StrictList, yaml.representer.RoundTripRepresenter.represent_list) +yaml.representer.RoundTripRepresenter.add_representer( + StrictDict, yaml.representer.RoundTripRepresenter.represent_dict) + +# Without this, ruamel.yaml will output "!!omap" types, which is +# technically correct but unnecessarily verbose for our uses +yaml.representer.RoundTripRepresenter.add_representer( + OrderedDict, yaml.representer.RoundTripRepresenter.represent_dict) + + +class JsonAsRawEncoder(json.JSONEncoder): + """ + A :class:`JSONEncoder` that will use the :code:`as_raw` property of objects + if available. + """ + def raw_encoder_default(self, obj): + try: + return iter(obj) + except TypeError: + if hasattr(obj, 'as_raw'): + return as_raw(obj) + return str(obj) + return super(JsonAsRawEncoder, self).default(obj) + + def __init__(self, *args, **kwargs): + kwargs['default'] = self.raw_encoder_default + super(JsonAsRawEncoder, self).__init__(*args, **kwargs) + + +class YamlAsRawDumper(yaml.dumper.RoundTripDumper): # pylint: disable=too-many-ancestors + """ + A :class:`RoundTripDumper` that will use the :code:`as_raw` property of objects + if available. + """ + + def represent_data(self, data): + if hasattr(data, 'as_raw'): + data = as_raw(data) + return super(YamlAsRawDumper, self).represent_data(data) + + +def full_type_name(value): + """ + The full class name of a type or object. + """ + + if not isinstance(value, type): + value = value.__class__ + module = str(value.__module__) + name = str(value.__name__) + return name if module == '__builtin__' else '%s.%s' % (module, name) + + +def safe_str(value): + """ + Like :code:`str` coercion, but makes sure that Unicode strings are properly + encoded, and will never return None. + """ + + try: + return str(value) + except UnicodeEncodeError: + return unicode(value).encode('utf8') + + +def safe_repr(value): + """ + Like :code:`repr`, but calls :code:`as_raw` and :code:`as_agnostic` first. + """ + + return repr(as_agnostic(as_raw(value))) + + +def string_list_as_string(strings): + """ + Nice representation of a list of strings. + """ + + return ', '.join('"%s"' % safe_str(v) for v in strings) + + +def as_raw(value): + """ + Converts values using their :code:`as_raw` property, if it exists, recursively. + """ + + if hasattr(value, 'as_raw'): + value = value.as_raw + if isinstance(value, MethodType): + # Old-style Python classes don't support properties + value = value() + elif isinstance(value, list): + value = list(value) + for i, _ in enumerate(value): + value[i] = as_raw(value[i]) + elif isinstance(value, dict): + value = dict(value) + for k, v in value.iteritems(): + value[k] = as_raw(v) + return value + + +def as_raw_list(value): + """ + Assuming value is a list, converts its values using :code:`as_raw`. + """ + + if value is None: + return [] + if isinstance(value, dict): + value = value.itervalues() + return [as_raw(v) for v in value] + + +def as_raw_dict(value): + """ + Assuming value is a dict, converts its values using :code:`as_raw`. + The keys are left as is. + """ + + if value is None: + return OrderedDict() + return OrderedDict(( + (k, as_raw(v)) for k, v in value.iteritems())) + + +def as_agnostic(value): + """ + Converts subclasses of list and dict to standard lists and dicts, and Unicode strings + to non-Unicode if possible, recursively. + + Useful for creating human-readable output of structures. + """ + + if isinstance(value, unicode): + try: + value = str(value) + except UnicodeEncodeError: + pass + elif isinstance(value, list): + value = list(value) + elif isinstance(value, dict): + value = dict(value) + + if isinstance(value, list): + for i, _ in enumerate(value): + value[i] = as_agnostic(value[i]) + elif isinstance(value, dict): + for k, v in value.iteritems(): + value[k] = as_agnostic(v) + + return value + + +def json_dumps(value, indent=2): + """ + JSON dumps that supports Unicode and the :code:`as_raw` property of objects + if available. + """ + + return json.dumps(value, indent=indent, ensure_ascii=False, cls=JsonAsRawEncoder) + + +def yaml_dumps(value, indent=2): + """ + YAML dumps that supports Unicode and the :code:`as_raw` property of objects + if available. + """ + + return yaml.dump(value, indent=indent, allow_unicode=True, Dumper=YamlAsRawDumper) + + +def yaml_loads(value): + return yaml.load(value, Loader=yaml.SafeLoader) http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/3895f8ca/aria/utils/imports.py ---------------------------------------------------------------------- diff --git a/aria/utils/imports.py b/aria/utils/imports.py new file mode 100644 index 0000000..e9c164e --- /dev/null +++ b/aria/utils/imports.py @@ -0,0 +1,78 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Utility methods for dynamically loading python code +""" + +import importlib + + +def import_fullname(name, paths=None): + """ + Imports a variable or class based on a full name, optionally searching for it in the paths. + """ + paths = paths or [] + if name is None: + return None + + def do_import(name): + if name and ('.' in name): + module_name, name = name.rsplit('.', 1) + return getattr(__import__(module_name, fromlist=[name], level=0), name) + else: + raise ImportError('import not found: %s' % name) + + try: + return do_import(name) + except ImportError: + for path in paths: + try: + return do_import('%s.%s' % (path, name)) + except Exception as e: + raise ImportError('cannot import %s, because %s' % (name, e)) + + raise ImportError('import not found: %s' % name) + + +def import_modules(name): + """ + Imports a module and all its sub-modules, recursively. + Relies on modules defining a 'MODULES' attribute listing their sub-module names. + """ + + module = __import__(name, fromlist=['MODULES'], level=0) + if hasattr(module, 'MODULES'): + for module_ in module.MODULES: + import_modules('%s.%s' % (name, module_)) + + +# TODO merge with import_fullname +def load_attribute(attribute_path): + """ + Dynamically load an attribute based on the path to it. + e.g. some_package.some_module.some_attribute, will load the some_attribute from the + some_package.some_module module + """ + module_name, attribute_name = attribute_path.rsplit('.', 1) + try: + module = importlib.import_module(module_name) + return getattr(module, attribute_name) + except ImportError: + # TODO: handle + raise + except AttributeError: + # TODO: handle + raise http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/3895f8ca/aria/utils/openclose.py ---------------------------------------------------------------------- diff --git a/aria/utils/openclose.py b/aria/utils/openclose.py new file mode 100644 index 0000000..19740eb --- /dev/null +++ b/aria/utils/openclose.py @@ -0,0 +1,32 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +class OpenClose(object): + """ + Wraps an object that has open() and close() methods to support the "with" keyword. + """ + + def __init__(self, wrapped): + self.wrapped = wrapped + + def __enter__(self): + if hasattr(self.wrapped, 'open'): + self.wrapped.open() + return self.wrapped + + def __exit__(self, the_type, value, traceback): + if hasattr(self.wrapped, 'close'): + self.wrapped.close() + return False http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/3895f8ca/aria/utils/plugin.py ---------------------------------------------------------------------- diff --git a/aria/utils/plugin.py b/aria/utils/plugin.py new file mode 100644 index 0000000..bb2b974 --- /dev/null +++ b/aria/utils/plugin.py @@ -0,0 +1,39 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Contains utility methods that enable dynamic python code loading +# TODO: merge with tools.module +""" + +import os +from importlib import import_module + + +def plugin_installer(path, plugin_suffix, package=None, callback=None): + """ + Load each module under ``path`` that ends with ``plugin_suffix``. If ``callback`` is supplied, + call it with each loaded module. + """ + assert callback is None or callable(callback) + plugin_suffix = '{0}.py'.format(plugin_suffix) + + for file_name in os.listdir(path): + if not file_name.endswith(plugin_suffix): + continue + module_name = '{0}.{1}'.format(package, file_name[:-3]) if package else file_name[:-3] + module = import_module(module_name) + if callback: + callback(module) http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/3895f8ca/aria/utils/threading.py ---------------------------------------------------------------------- diff --git a/aria/utils/threading.py b/aria/utils/threading.py new file mode 100644 index 0000000..575d011 --- /dev/null +++ b/aria/utils/threading.py @@ -0,0 +1,252 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import absolute_import # so we can import standard 'threading' + +import itertools +import multiprocessing +from threading import (Thread, Lock) +from Queue import (Queue, Full, Empty) + +from .exceptions import print_exception + +class ExecutorException(Exception): + pass + +class DaemonThread(Thread): + def __init__(self, *args, **kwargs): + super(DaemonThread, self).__init__(*args, **kwargs) + self.daemon = True + + def run(self): + """ + We're overriding `Thread.run` in order to avoid annoying (but harmless) error + messages during shutdown. The problem is that CPython nullifies the + global state _before_ shutting down daemon threads, so that exceptions + might happen, and then `Thread.__bootstrap_inner` prints them out. + + Our solution is to swallow these exceptions here. + + The side effect is that uncaught exceptions in our own thread code will _not_ + be printed out as usual, so it's our responsibility to catch them in our + code. + """ + + try: + super(DaemonThread, self).run() + except SystemExit as e: + # This exception should be bubbled up + raise e + except BaseException: + # Exceptions might occur in daemon threads during interpreter shutdown + pass + +# https://gist.github.com/tliron/81dd915166b0bfc64be08b4f8e22c835 +class FixedThreadPoolExecutor(object): + """ + Executes tasks in a fixed thread pool. + + Makes sure to gather all returned results and thrown exceptions in one place, in order of task + submission. + + Example:: + + def sum(arg1, arg2): + return arg1 + arg2 + + executor = FixedThreadPoolExecutor(10) + try: + for value in range(100): + executor.submit(sum, value, value) + executor.drain() + except: + executor.close() + executor.raise_first() + print executor.returns + + You can also use it with the Python "with" keyword, in which case you don't need to call "close" + explicitly:: + + with FixedThreadPoolExecutor(10) as executor: + for value in range(100): + executor.submit(sum, value, value) + executor.drain() + executor.raise_first() + print executor.returns + """ + + _CYANIDE = object() # Special task marker used to kill worker threads. + + def __init__(self, + size=multiprocessing.cpu_count() * 2 + 1, + timeout=None, + print_exceptions=False): + """ + :param size: Number of threads in the pool (fixed). + :param timeout: Timeout in seconds for all + blocking operations. (Defaults to none, meaning no timeout) + :param print_exceptions: Set to true in order to + print exceptions from tasks. (Defaults to false) + """ + + self.size = size + self.timeout = timeout + self.print_exceptions = print_exceptions + + self._tasks = Queue() + self._returns = {} + self._exceptions = {} + self._id_creator = itertools.count() + self._lock = Lock() # for console output + + self._workers = [] + for index in range(size): + worker = DaemonThread( + name='%s%d' % (self.__class__.__name__, index), + target=self._thread_worker) + worker.start() + self._workers.append(worker) + + def submit(self, func, *args, **kwargs): + """ + Submit a task for execution. + + The task will be called ASAP on the next available worker thread in the pool. + + Will raise an :class:`ExecutorException` exception if cannot be submitted. + """ + + try: + self._tasks.put((self._id_creator.next(), func, args, kwargs), timeout=self.timeout) + except Full: + raise ExecutorException('cannot submit task: queue is full') + + def close(self): + """ + Blocks until all current tasks finish execution and all worker threads are dead. + + You cannot submit tasks anymore after calling this. + + This is called automatically upon exit if you are using the "with" keyword. + """ + + self.drain() + while self.is_alive: + try: + self._tasks.put(self._CYANIDE, timeout=self.timeout) + except Full: + raise ExecutorException('cannot close executor: a thread seems to be hanging') + self._workers = None + + def drain(self): + """ + Blocks until all current tasks finish execution, but leaves the worker threads alive. + """ + + self._tasks.join() # oddly, the API does not support a timeout parameter + + @property + def is_alive(self): + """ + True if any of the worker threads are alive. + """ + + for worker in self._workers: + if worker.is_alive(): + return True + return False + + @property + def returns(self): + """ + The returned values from all tasks, in order of submission. + """ + + return [self._returns[k] for k in sorted(self._returns)] + + @property + def exceptions(self): + """ + The raised exceptions from all tasks, in order of submission. + """ + + return [self._exceptions[k] for k in sorted(self._exceptions)] + + def raise_first(self): + """ + If exceptions were thrown by any task, then the first one will be raised. + + This is rather arbitrary: proper handling would involve iterating all the + exceptions. However, if you want to use the "raise" mechanism, you are + limited to raising only one of them. + """ + + exceptions = self.exceptions + if exceptions: + raise exceptions[0] + + def _thread_worker(self): + while True: + if not self._execute_next_task(): + break + + def _execute_next_task(self): + try: + task = self._tasks.get(timeout=self.timeout) + except Empty: + # Happens if timeout is reached + return True + if task == self._CYANIDE: + # Time to die :( + return False + self._execute_task(*task) + return True + + def _execute_task(self, task_id, func, args, kwargs): + try: + result = func(*args, **kwargs) + self._returns[task_id] = result + except Exception as e: + self._exceptions[task_id] = e + if self.print_exceptions: + with self._lock: + print_exception(e) + self._tasks.task_done() + + def __enter__(self): + return self + + def __exit__(self, the_type, value, traceback): + self.close() + return False + +class LockedList(list): + """ + A list that supports the "with" keyword with a built-in lock. + + Though Python lists are thread-safe in that they will not raise exceptions + during concurrent access, they do not guarantee atomicity. This class will + let you gain atomicity when needed. + """ + + def __init__(self, *args, **kwargs): + super(LockedList, self).__init__(*args, **kwargs) + self.lock = Lock() + + def __enter__(self): + return self.lock.__enter__() + + def __exit__(self, the_type, value, traceback): + return self.lock.__exit__(the_type, value, traceback) http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/3895f8ca/aria/utils/uris.py ---------------------------------------------------------------------- diff --git a/aria/utils/uris.py b/aria/utils/uris.py new file mode 100644 index 0000000..1686517 --- /dev/null +++ b/aria/utils/uris.py @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import urlparse + +def as_file(uri): + """ + If the URI is a file (either the :code:`file` scheme or no scheme), then returns the absolute + path. Otherwise, returns None. + """ + + url = urlparse.urlparse(uri) + if (not url.scheme) or (url.scheme == 'file'): + return os.path.abspath(url.path) + return None http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/3895f8ca/aria/utils/validation.py ---------------------------------------------------------------------- diff --git a/aria/utils/validation.py b/aria/utils/validation.py new file mode 100644 index 0000000..a33f7a2 --- /dev/null +++ b/aria/utils/validation.py @@ -0,0 +1,95 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Contains validation related utilities +""" + + +class ValidatorMixin(object): + """ + A mixin that should be added to classes that require validating user input + """ + + _ARGUMENT_TYPE_MESSAGE = '{name} argument must be {type} based, got {arg!r}' + _ARGUMENT_CHOICE_MESSAGE = '{name} argument must be in {choices}, got {arg!r}' + + @classmethod + def validate_in_choice(cls, name, argument, choices): + """ + Validate ``argument`` is in ``choices`` + """ + if argument not in choices: + raise TypeError(cls._ARGUMENT_CHOICE_MESSAGE.format( + name=name, choices=choices, arg=argument)) + + @classmethod + def validate_type(cls, argument_name, argument, expected_type): + """ + Validate ``argument`` is a subclass of ``expected_type`` + """ + if not issubclass(argument, expected_type): + raise TypeError(cls._ARGUMENT_TYPE_MESSAGE.format( + name=argument_name, type=expected_type, arg=argument)) + + @classmethod + def validate_instance(cls, argument_name, argument, expected_type): + """ + Validate ``argument`` is a instance of ``expected_type`` + """ + if not isinstance(argument, expected_type): + raise TypeError(cls._ARGUMENT_TYPE_MESSAGE.format( + name=argument_name, type=expected_type, arg=argument)) + + @classmethod + def validate_callable(cls, argument_name, argument): + """ + Validate ``argument`` is callable + """ + if not callable(argument): + raise TypeError(cls._ARGUMENT_TYPE_MESSAGE.format( + name=argument_name, type='callable', arg=argument)) + + +def validate_function_arguments(func, func_kwargs): + """ + Validates all required arguments are supplied to ``func`` and that no additional arguments are + supplied + """ + + _kwargs_flags = 8 + + has_kwargs = func.func_code.co_flags & _kwargs_flags != 0 + args_count = func.func_code.co_argcount + + # all args without the ones with default values + args = func.func_code.co_varnames[:args_count] + non_default_args = args[:len(func.func_defaults)] if func.func_defaults else args + + # Check if any args without default values is missing in the func_kwargs + for arg in non_default_args: + if arg not in func_kwargs: + raise ValueError( + "The argument '{arg}' doest not have a default value, and it " + "isn't passed to {func.__name__}".format(arg=arg, func=func)) + + # check if there are any extra kwargs + extra_kwargs = [arg for arg in func_kwargs.keys() if arg not in args] + + # assert that the function has kwargs + if extra_kwargs and not has_kwargs: + raise ValueError("The following extra kwargs were supplied: {extra_kwargs}".format( + extra_kwargs=extra_kwargs + )) http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/3895f8ca/extensions/aria_extension_tosca/simple_nfv_v1_0/presenter.py ---------------------------------------------------------------------- diff --git a/extensions/aria_extension_tosca/simple_nfv_v1_0/presenter.py b/extensions/aria_extension_tosca/simple_nfv_v1_0/presenter.py index e26ca7b..8098ccf 100644 --- a/extensions/aria_extension_tosca/simple_nfv_v1_0/presenter.py +++ b/extensions/aria_extension_tosca/simple_nfv_v1_0/presenter.py @@ -13,10 +13,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -from aria.parser.utils import (FrozenList, EMPTY_READ_ONLY_LIST, cachedmethod) +from aria.utils.collections import FrozenList, EMPTY_READ_ONLY_LIST +from aria.utils.caching import cachedmethod from ..simple_v1_0 import ToscaSimplePresenter1_0 + class ToscaSimpleNfvPresenter1_0(ToscaSimplePresenter1_0): # pylint: disable=invalid-name """ ARIA presenter for the `TOSCA Simple Profile for NFV v1.0 csd03 <http://docs.oasis-open.org http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/3895f8ca/extensions/aria_extension_tosca/simple_v1_0/assignments.py ---------------------------------------------------------------------- diff --git a/extensions/aria_extension_tosca/simple_v1_0/assignments.py b/extensions/aria_extension_tosca/simple_v1_0/assignments.py index 6f42d79..14fee96 100644 --- a/extensions/aria_extension_tosca/simple_v1_0/assignments.py +++ b/extensions/aria_extension_tosca/simple_v1_0/assignments.py @@ -13,12 +13,13 @@ # See the License for the specific language governing permissions and # limitations under the License. +from aria.utils.collections import FrozenDict +from aria.utils.caching import cachedmethod from aria.parser import dsl_specification from aria.parser.presentation import (AsIsPresentation, has_fields, allow_unknown_fields, short_form_field, primitive_field, object_field, object_dict_field, object_dict_unknown_fields, field_validator, type_validator) -from aria.parser.utils import FrozenDict, cachedmethod from .filters import NodeFilter from .misc import Description, OperationImplementation http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/3895f8ca/extensions/aria_extension_tosca/simple_v1_0/data_types.py ---------------------------------------------------------------------- diff --git a/extensions/aria_extension_tosca/simple_v1_0/data_types.py b/extensions/aria_extension_tosca/simple_v1_0/data_types.py index e9cd97c..1fdbe6e 100644 --- a/extensions/aria_extension_tosca/simple_v1_0/data_types.py +++ b/extensions/aria_extension_tosca/simple_v1_0/data_types.py @@ -14,19 +14,17 @@ # limitations under the License. import re -try: - from collections import OrderedDict -except ImportError: - from ordereddict import OrderedDict from functools import total_ordering from datetime import datetime, tzinfo, timedelta from aria.parser import dsl_specification -from aria.parser.utils import StrictDict, safe_repr +from aria.utils.collections import StrictDict, OrderedDict +from aria.utils.formatting import safe_repr from .modeling.data_types import (coerce_to_data_type_class, report_issue_for_bad_format, coerce_value) + class Timezone(tzinfo): """ Timezone as fixed offset in hours and minutes east of UTC. http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/3895f8ca/extensions/aria_extension_tosca/simple_v1_0/definitions.py ---------------------------------------------------------------------- diff --git a/extensions/aria_extension_tosca/simple_v1_0/definitions.py b/extensions/aria_extension_tosca/simple_v1_0/definitions.py index e595e07..b60a797 100644 --- a/extensions/aria_extension_tosca/simple_v1_0/definitions.py +++ b/extensions/aria_extension_tosca/simple_v1_0/definitions.py @@ -13,8 +13,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +from aria.utils.collections import FrozenDict +from aria.utils.caching import cachedmethod from aria.parser import dsl_specification -from aria.parser.utils import (FrozenDict, cachedmethod) from aria.parser.presentation import (has_fields, short_form_field, allow_unknown_fields, primitive_field, primitive_list_field, object_field, object_list_field, object_dict_field, http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/3895f8ca/extensions/aria_extension_tosca/simple_v1_0/filters.py ---------------------------------------------------------------------- diff --git a/extensions/aria_extension_tosca/simple_v1_0/filters.py b/extensions/aria_extension_tosca/simple_v1_0/filters.py index 7ead633..617ce7a 100644 --- a/extensions/aria_extension_tosca/simple_v1_0/filters.py +++ b/extensions/aria_extension_tosca/simple_v1_0/filters.py @@ -13,8 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from aria.utils.caching import cachedmethod from aria.parser import dsl_specification -from aria.parser.utils import cachedmethod from aria.parser.presentation import (has_fields, object_sequenced_list_field, field_validator) from .misc import ConstraintClause http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/3895f8ca/extensions/aria_extension_tosca/simple_v1_0/functions.py ---------------------------------------------------------------------- diff --git a/extensions/aria_extension_tosca/simple_v1_0/functions.py b/extensions/aria_extension_tosca/simple_v1_0/functions.py index bf2f69d..79079df 100644 --- a/extensions/aria_extension_tosca/simple_v1_0/functions.py +++ b/extensions/aria_extension_tosca/simple_v1_0/functions.py @@ -15,9 +15,10 @@ from cStringIO import StringIO +from aria.utils.collections import FrozenList +from aria.utils.formatting import as_raw, safe_repr from aria.parser import (dsl_specification, InvalidValueError) from aria.parser.modeling import (Function, CannotEvaluateFunctionException) -from aria.parser.utils import (FrozenList, as_raw, safe_repr) from aria.parser.validation import Issue # http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/3895f8ca/extensions/aria_extension_tosca/simple_v1_0/misc.py ---------------------------------------------------------------------- diff --git a/extensions/aria_extension_tosca/simple_v1_0/misc.py b/extensions/aria_extension_tosca/simple_v1_0/misc.py index 5e41378..4453c7a 100644 --- a/extensions/aria_extension_tosca/simple_v1_0/misc.py +++ b/extensions/aria_extension_tosca/simple_v1_0/misc.py @@ -13,13 +13,15 @@ # See the License for the specific language governing permissions and # limitations under the License. +from aria.utils.caching import cachedmethod +from aria.utils.console import puts +from aria.utils.formatting import as_raw from aria.parser import dsl_specification from aria.parser.presentation import (AsIsPresentation, has_fields, allow_unknown_fields, short_form_field, primitive_field, primitive_list_field, primitive_dict_unknown_fields, object_field, object_list_field, object_dict_field, field_validator, type_validator) -from aria.parser.utils import (cachedmethod, puts, as_raw) from .modeling.data_types import (get_data_type, get_data_type_value, get_property_constraints, apply_constraint_to_value) http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/3895f8ca/extensions/aria_extension_tosca/simple_v1_0/modeling/artifacts.py ---------------------------------------------------------------------- diff --git a/extensions/aria_extension_tosca/simple_v1_0/modeling/artifacts.py b/extensions/aria_extension_tosca/simple_v1_0/modeling/artifacts.py index f5667a6..4f61ef5 100644 --- a/extensions/aria_extension_tosca/simple_v1_0/modeling/artifacts.py +++ b/extensions/aria_extension_tosca/simple_v1_0/modeling/artifacts.py @@ -13,15 +13,13 @@ # See the License for the specific language governing permissions and # limitations under the License. -try: - from collections import OrderedDict -except ImportError: - from ordereddict import OrderedDict +from aria.utils.collections import OrderedDict # # NodeType, NodeTemplate # + def get_inherited_artifact_definitions(context, presentation, for_presentation=None): if hasattr(presentation, '_get_type'): http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/3895f8ca/extensions/aria_extension_tosca/simple_v1_0/modeling/capabilities.py ---------------------------------------------------------------------- diff --git a/extensions/aria_extension_tosca/simple_v1_0/modeling/capabilities.py b/extensions/aria_extension_tosca/simple_v1_0/modeling/capabilities.py index 4a96b04..d9b9f6b 100644 --- a/extensions/aria_extension_tosca/simple_v1_0/modeling/capabilities.py +++ b/extensions/aria_extension_tosca/simple_v1_0/modeling/capabilities.py @@ -13,12 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -try: - from collections import OrderedDict -except ImportError: - from ordereddict import OrderedDict - -from aria.parser.utils import deepcopy_with_locators +from aria.utils.collections import deepcopy_with_locators, OrderedDict from aria.parser.validation import Issue from .properties import (convert_property_definitions_to_values, merge_raw_property_definitions, http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/3895f8ca/extensions/aria_extension_tosca/simple_v1_0/modeling/data_types.py ---------------------------------------------------------------------- diff --git a/extensions/aria_extension_tosca/simple_v1_0/modeling/data_types.py b/extensions/aria_extension_tosca/simple_v1_0/modeling/data_types.py index d98bc54..99dcfea 100644 --- a/extensions/aria_extension_tosca/simple_v1_0/modeling/data_types.py +++ b/extensions/aria_extension_tosca/simple_v1_0/modeling/data_types.py @@ -14,14 +14,12 @@ # limitations under the License. import re -try: - from collections import OrderedDict -except ImportError: - from ordereddict import OrderedDict +from aria.utils.collections import OrderedDict +from aria.utils.formatting import full_type_name, safe_repr +from aria.utils.imports import import_fullname from aria.parser import dsl_specification from aria.parser.presentation import (get_locator, validate_primitive) -from aria.parser.utils import (import_fullname, full_type_name, safe_repr) from aria.parser.validation import Issue from ..functions import get_function http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/3895f8ca/extensions/aria_extension_tosca/simple_v1_0/modeling/interfaces.py ---------------------------------------------------------------------- diff --git a/extensions/aria_extension_tosca/simple_v1_0/modeling/interfaces.py b/extensions/aria_extension_tosca/simple_v1_0/modeling/interfaces.py index bffb5dc..3e6aa6f 100644 --- a/extensions/aria_extension_tosca/simple_v1_0/modeling/interfaces.py +++ b/extensions/aria_extension_tosca/simple_v1_0/modeling/interfaces.py @@ -13,13 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -try: - from collections import OrderedDict -except ImportError: - from ordereddict import OrderedDict - +from aria.utils.collections import merge, deepcopy_with_locators, OrderedDict from aria.parser.presentation import get_locator -from aria.parser.utils import (merge, deepcopy_with_locators) from aria.parser.validation import Issue from .properties import (coerce_property_value, convert_property_definitions_to_values) http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/3895f8ca/extensions/aria_extension_tosca/simple_v1_0/modeling/properties.py ---------------------------------------------------------------------- diff --git a/extensions/aria_extension_tosca/simple_v1_0/modeling/properties.py b/extensions/aria_extension_tosca/simple_v1_0/modeling/properties.py index 439f44c..f61cb99 100644 --- a/extensions/aria_extension_tosca/simple_v1_0/modeling/properties.py +++ b/extensions/aria_extension_tosca/simple_v1_0/modeling/properties.py @@ -13,13 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -try: - from collections import OrderedDict -except ImportError: - from ordereddict import OrderedDict - +from aria.utils.collections import merge, deepcopy_with_locators, OrderedDict from aria.parser.presentation import Value -from aria.parser.utils import (merge, deepcopy_with_locators) from aria.parser.validation import Issue from .data_types import coerce_value http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/3895f8ca/extensions/aria_extension_tosca/simple_v1_0/modeling/requirements.py ---------------------------------------------------------------------- diff --git a/extensions/aria_extension_tosca/simple_v1_0/modeling/requirements.py b/extensions/aria_extension_tosca/simple_v1_0/modeling/requirements.py index 068dde0..9dea874 100644 --- a/extensions/aria_extension_tosca/simple_v1_0/modeling/requirements.py +++ b/extensions/aria_extension_tosca/simple_v1_0/modeling/requirements.py @@ -13,12 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -try: - from collections import OrderedDict -except ImportError: - from ordereddict import OrderedDict from aria.parser.validation import Issue -from aria.parser.utils import deepcopy_with_locators +from aria.utils.collections import deepcopy_with_locators, OrderedDict from .properties import (convert_property_definitions_to_values, validate_required_values, coerce_property_value) http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/3895f8ca/extensions/aria_extension_tosca/simple_v1_0/modeling/substitution_mappings.py ---------------------------------------------------------------------- diff --git a/extensions/aria_extension_tosca/simple_v1_0/modeling/substitution_mappings.py b/extensions/aria_extension_tosca/simple_v1_0/modeling/substitution_mappings.py index e0252b1..c1e21de 100644 --- a/extensions/aria_extension_tosca/simple_v1_0/modeling/substitution_mappings.py +++ b/extensions/aria_extension_tosca/simple_v1_0/modeling/substitution_mappings.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from aria.parser.utils import safe_repr +from aria.utils.formatting import safe_repr from aria.parser.validation import Issue def validate_subtitution_mappings_requirement(context, presentation): http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/3895f8ca/extensions/aria_extension_tosca/simple_v1_0/presentation/extensible.py ---------------------------------------------------------------------- diff --git a/extensions/aria_extension_tosca/simple_v1_0/presentation/extensible.py b/extensions/aria_extension_tosca/simple_v1_0/presentation/extensible.py index 22e9606..a2fd6ee 100644 --- a/extensions/aria_extension_tosca/simple_v1_0/presentation/extensible.py +++ b/extensions/aria_extension_tosca/simple_v1_0/presentation/extensible.py @@ -13,8 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from aria.utils.caching import cachedmethod from aria.parser.presentation import (Presentation, has_fields, primitive_dict_field) -from aria.parser.utils import cachedmethod @has_fields class ExtensiblePresentation(Presentation): http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/3895f8ca/extensions/aria_extension_tosca/simple_v1_0/presentation/field_getters.py ---------------------------------------------------------------------- diff --git a/extensions/aria_extension_tosca/simple_v1_0/presentation/field_getters.py b/extensions/aria_extension_tosca/simple_v1_0/presentation/field_getters.py index 291e349..a2adb1e 100644 --- a/extensions/aria_extension_tosca/simple_v1_0/presentation/field_getters.py +++ b/extensions/aria_extension_tosca/simple_v1_0/presentation/field_getters.py @@ -13,8 +13,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +from aria.utils.formatting import safe_repr from aria.parser import InvalidValueError -from aria.parser.utils import safe_repr + def data_type_class_getter(cls): """ http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/3895f8ca/extensions/aria_extension_tosca/simple_v1_0/presentation/field_validators.py ---------------------------------------------------------------------- diff --git a/extensions/aria_extension_tosca/simple_v1_0/presentation/field_validators.py b/extensions/aria_extension_tosca/simple_v1_0/presentation/field_validators.py index 662faab..f1b0a20 100644 --- a/extensions/aria_extension_tosca/simple_v1_0/presentation/field_validators.py +++ b/extensions/aria_extension_tosca/simple_v1_0/presentation/field_validators.py @@ -15,9 +15,9 @@ import re +from aria.utils.formatting import safe_repr from aria.parser import dsl_specification from aria.parser.presentation import (report_issue_for_unknown_type, derived_from_validator) -from aria.parser.utils import safe_repr from aria.parser.validation import Issue from ..modeling.data_types import (get_primitive_data_type, get_data_type_name, coerce_value, http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/3895f8ca/extensions/aria_extension_tosca/simple_v1_0/presenter.py ---------------------------------------------------------------------- diff --git a/extensions/aria_extension_tosca/simple_v1_0/presenter.py b/extensions/aria_extension_tosca/simple_v1_0/presenter.py index c21c326..0809014 100644 --- a/extensions/aria_extension_tosca/simple_v1_0/presenter.py +++ b/extensions/aria_extension_tosca/simple_v1_0/presenter.py @@ -13,8 +13,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +from aria.utils.collections import FrozenList, EMPTY_READ_ONLY_LIST +from aria.utils.caching import cachedmethod from aria.parser.presentation import Presenter -from aria.parser.utils import (FrozenList, EMPTY_READ_ONLY_LIST, cachedmethod) from .functions import (Concat, Token, GetInput, GetProperty, GetAttribute, GetOperationOutput, GetNodesOfType, GetArtifact) http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/3895f8ca/extensions/aria_extension_tosca/simple_v1_0/templates.py ---------------------------------------------------------------------- diff --git a/extensions/aria_extension_tosca/simple_v1_0/templates.py b/extensions/aria_extension_tosca/simple_v1_0/templates.py index fda95a4..6860b72 100644 --- a/extensions/aria_extension_tosca/simple_v1_0/templates.py +++ b/extensions/aria_extension_tosca/simple_v1_0/templates.py @@ -13,12 +13,13 @@ # See the License for the specific language governing permissions and # limitations under the License. +from aria.utils.collections import FrozenDict, FrozenList +from aria.utils.caching import cachedmethod from aria.parser import dsl_specification from aria.parser.presentation import (has_fields, primitive_field, primitive_list_field, object_field, object_list_field, object_dict_field, object_sequenced_list_field, field_validator, type_validator, list_type_validator) -from aria.parser.utils import (FrozenDict, FrozenList, cachedmethod) from .assignments import (PropertyAssignment, AttributeAssignment, RequirementAssignment, CapabilityAssignment, InterfaceAssignment, ArtifactAssignment) http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/3895f8ca/extensions/aria_extension_tosca/simple_v1_0/types.py ---------------------------------------------------------------------- diff --git a/extensions/aria_extension_tosca/simple_v1_0/types.py b/extensions/aria_extension_tosca/simple_v1_0/types.py index 37608a2..39843ac 100644 --- a/extensions/aria_extension_tosca/simple_v1_0/types.py +++ b/extensions/aria_extension_tosca/simple_v1_0/types.py @@ -13,6 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from aria.utils.collections import FrozenDict, FrozenList +from aria.utils.caching import cachedmethod from aria.parser import dsl_specification from aria.parser.presentation import (has_fields, allow_unknown_fields, primitive_field, primitive_list_field, object_field, object_dict_field, @@ -20,7 +22,6 @@ from aria.parser.presentation import (has_fields, allow_unknown_fields, primitiv object_dict_unknown_fields, field_getter, field_validator, list_type_validator, derived_from_validator, get_parent_presentation) -from aria.parser.utils import (FrozenDict, FrozenList, cachedmethod) from .assignments import ArtifactAssignment from .data_types import Version http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/3895f8ca/requirements.txt ---------------------------------------------------------------------- diff --git a/requirements.txt b/requirements.txt index a9a7963..e6d5393 100644 --- a/requirements.txt +++ b/requirements.txt @@ -23,4 +23,3 @@ Jinja2==2.8 shortuuid==0.4.3 CacheControl[filecache]==0.11.6 clint==0.5.1 -python-daemon==2.1.2 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/3895f8ca/setup.py ---------------------------------------------------------------------- diff --git a/setup.py b/setup.py index 14c5873..112f13e 100644 --- a/setup.py +++ b/setup.py @@ -16,22 +16,29 @@ import os import sys + from setuptools import setup, find_packages _PACKAGE_NAME = 'aria' _PYTHON_SUPPORTED_VERSIONS = [(2, 6), (2, 7)] +_EXTENSION_DIR = 'extensions' +_EXTENSION_NAMES = [ + 'aria_extension_tosca' +] if (sys.version_info[0], sys.version_info[1]) not in _PYTHON_SUPPORTED_VERSIONS: raise NotImplementedError( '{0} Package support Python version 2.6 & 2.7 Only'.format( _PACKAGE_NAME)) +root_dir = os.path.dirname(__file__) + version = '0.1.0' -execfile(os.path.join('.', _PACKAGE_NAME, 'VERSION.py')) +execfile(os.path.join(root_dir, _PACKAGE_NAME, 'VERSION.py')) try: - with open('./requirements.txt') as requirements: + with open(os.path.join(root_dir, 'requirements.txt')) as requirements: install_requires = [requirement.strip() for requirement in requirements.readlines()] except IOError: install_requires = [] @@ -45,7 +52,6 @@ setup( author='aria', author_email='[email protected]', url='http://ariatosca.org', - classifiers=[ 'Development Status :: 4 - Beta', 'Environment :: Console', @@ -58,17 +64,16 @@ setup( 'Topic :: Software Development :: Libraries :: Python Modules', 'Topic :: System :: Networking', 'Topic :: System :: Systems Administration'], - - packages=find_packages(exclude=('*tests*',)), + packages=find_packages(include=['aria*']) + + find_packages(where=_EXTENSION_DIR, + include=['{0}*'.format(name) for name in _EXTENSION_NAMES]), + package_dir=dict((name, '{0}/{1}'.format(_EXTENSION_DIR, name)) for name in _EXTENSION_NAMES), package_data={ - 'aria.tools': [ - 'web/**'], 'aria_extension_tosca': [ 'profiles/tosca-simple-1.0/**', - 'profiles/tosca-simple-nfv-1.0/**'], - 'aria_extension_open_o': [ - 'web/**']}, - + 'profiles/tosca-simple-nfv-1.0/**' + ] + }, zip_safe=False, install_requires=install_requires, entry_points={ http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/3895f8ca/tests/orchestrator/context/test_workflow.py ---------------------------------------------------------------------- diff --git a/tests/orchestrator/context/test_workflow.py b/tests/orchestrator/context/test_workflow.py index 19eb57c..258f0c5 100644 --- a/tests/orchestrator/context/test_workflow.py +++ b/tests/orchestrator/context/test_workflow.py @@ -59,5 +59,6 @@ class TestWorkflowContext(object): def storage(): result = application_model_storage(InMemoryModelDriver()) result.setup() + result.blueprint.store(models.get_blueprint()) result.deployment.store(models.get_deployment()) return result
