http://git-wip-us.apache.org/repos/asf/incubator-ariatosca-website/blob/23d6ba76/apache-ariatosca-0.1.1/aria/parser/reading/context.py ---------------------------------------------------------------------- diff --git a/apache-ariatosca-0.1.1/aria/parser/reading/context.py b/apache-ariatosca-0.1.1/aria/parser/reading/context.py deleted file mode 100644 index 233e407..0000000 --- a/apache-ariatosca-0.1.1/aria/parser/reading/context.py +++ /dev/null @@ -1,31 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from ...utils.threading import LockedList -from .source import DefaultReaderSource - - -class ReadingContext(object): - """ - Reading context. - - :ivar reader_source: for finding reader instances - :vartype reader_source: ReaderSource - :ivar reader: overrides ``reader_source`` with a specific class - :vartype reader: type - """ - - def __init__(self): - self.reader_source = DefaultReaderSource() - self.reader = None - - self._locations = LockedList() # for keeping track of locations already read
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca-website/blob/23d6ba76/apache-ariatosca-0.1.1/aria/parser/reading/exceptions.py ---------------------------------------------------------------------- diff --git a/apache-ariatosca-0.1.1/aria/parser/reading/exceptions.py b/apache-ariatosca-0.1.1/aria/parser/reading/exceptions.py deleted file mode 100644 index 3699729..0000000 --- a/apache-ariatosca-0.1.1/aria/parser/reading/exceptions.py +++ /dev/null @@ -1,44 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from ...exceptions import AriaException -from ..validation import Issue - - -class ReaderException(AriaException): - """ - ARIA reader exception. - """ - - -class ReaderNotFoundError(ReaderException): - """ - ARIA reader error: reader not found for source. - """ - - -class ReaderSyntaxError(ReaderException): - """ - ARIA read format error. - """ - - def __init__(self, message, cause=None, cause_tb=None, location=None, line=None, - column=None, locator=None, snippet=None, level=Issue.SYNTAX): - super(ReaderSyntaxError, self).__init__(message, cause, cause_tb) - self.issue = Issue(message, location=location, line=line, column=column, - locator=locator, snippet=snippet, level=level) - - -class AlreadyReadException(ReaderException): - """ - ARIA reader exception: already read. - """ http://git-wip-us.apache.org/repos/asf/incubator-ariatosca-website/blob/23d6ba76/apache-ariatosca-0.1.1/aria/parser/reading/jinja.py ---------------------------------------------------------------------- diff --git a/apache-ariatosca-0.1.1/aria/parser/reading/jinja.py b/apache-ariatosca-0.1.1/aria/parser/reading/jinja.py deleted file mode 100644 index 687317a..0000000 --- a/apache-ariatosca-0.1.1/aria/parser/reading/jinja.py +++ /dev/null @@ -1,55 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os - -from jinja2 import Template - -from ... import __version__ as version -from ..loading import LiteralLocation, LiteralLoader -from .reader import Reader -from .exceptions import ReaderSyntaxError - - -# TODO: we could put a lot of other useful stuff here. -CONTEXT = { - 'ARIA_VERSION': version, - 'ENV': os.environ} - - -class JinjaReader(Reader): - """ - ARIA Jinja reader. - - Forwards the rendered result to a new reader in the reader source. - """ - - def read(self): - data = self.load() - try: - data = str(data) - template = Template(data) - literal = template.render(CONTEXT) - # TODO: might be useful to write the literal result to a file for debugging - location = self.location - if isinstance(location, basestring) and location.endswith('.jinja'): - # Use reader based on the location with the ".jinja" prefix stripped off - location = location[:-6] - next_reader = self.context.reading.reader_source.get_reader( - self.context, LiteralLocation(literal, name=location), LiteralLoader(literal)) - else: - # Use reader for literal loader - next_reader = self.context.reading.reader_source.get_reader( - self.context, LiteralLocation(literal), LiteralLoader(literal)) - return next_reader.read() - except Exception as e: - raise ReaderSyntaxError('Jinja: %s' % e, cause=e) http://git-wip-us.apache.org/repos/asf/incubator-ariatosca-website/blob/23d6ba76/apache-ariatosca-0.1.1/aria/parser/reading/json.py ---------------------------------------------------------------------- diff --git a/apache-ariatosca-0.1.1/aria/parser/reading/json.py b/apache-ariatosca-0.1.1/aria/parser/reading/json.py deleted file mode 100644 index d144f80..0000000 --- a/apache-ariatosca-0.1.1/aria/parser/reading/json.py +++ /dev/null @@ -1,33 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from __future__ import absolute_import # so we can import standard 'json' - -import json - -from ...utils.collections import OrderedDict -from .reader import Reader -from .exceptions import ReaderSyntaxError - - -class JsonReader(Reader): - """ - ARIA JSON reader. - """ - - def read(self): - data = self.load() - try: - data = unicode(data) - return json.loads(data, object_pairs_hook=OrderedDict) - except Exception as e: - raise ReaderSyntaxError('JSON: %s' % e, cause=e) http://git-wip-us.apache.org/repos/asf/incubator-ariatosca-website/blob/23d6ba76/apache-ariatosca-0.1.1/aria/parser/reading/locator.py ---------------------------------------------------------------------- diff --git a/apache-ariatosca-0.1.1/aria/parser/reading/locator.py b/apache-ariatosca-0.1.1/aria/parser/reading/locator.py deleted file mode 100644 index 965164d..0000000 --- a/apache-ariatosca-0.1.1/aria/parser/reading/locator.py +++ /dev/null @@ -1,154 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from copy import deepcopy - - -from ...utils.console import puts, Colored, indent - - -# We are inheriting the primitive types in order to add the ability to set -# an attribute (_locator) on them. - -class LocatableString(unicode): - pass - - -class LocatableInt(int): - pass - - -class LocatableFloat(float): - pass - - -def wrap(value): - if isinstance(value, basestring): - return True, LocatableString(value) - elif isinstance(value, int) and \ - not isinstance(value, bool): # Note: bool counts as int in Python! - return True, LocatableInt(value) - elif isinstance(value, float): - return True, LocatableFloat(value) - return False, value - - -class Locator(object): - """ - Stores location information (line and column numbers) for agnostic raw data. - """ - def __init__(self, location, line, column, children=None): - self.location = location - self.line = line - self.column = column - self.children = children - - def get_child(self, *names): - if (not names) or (not isinstance(self.children, dict)): - return self - name = names[0] - if name not in self.children: - return self - child = self.children[name] - return child.get_child(names[1:]) - - def link(self, raw, path=None): - if hasattr(raw, '_locator'): - # This can happen when we use anchors - return - - try: - setattr(raw, '_locator', self) - except AttributeError: - return - - if isinstance(raw, list): - for i, raw_element in enumerate(raw): - wrapped, raw_element = wrap(raw_element) - if wrapped: - raw[i] = raw_element - child_path = '%s.%d' % (path, i) if path else str(i) - try: - self.children[i].link(raw_element, child_path) - except KeyError: - raise ValueError('location map does not match agnostic raw data: %s' % - child_path) - elif isinstance(raw, dict): - for k, raw_element in raw.iteritems(): - wrapped, raw_element = wrap(raw_element) - if wrapped: - raw[k] = raw_element - child_path = '%s.%s' % (path, k) if path else k - try: - self.children[k].link(raw_element, child_path) - except KeyError: - raise ValueError('location map does not match agnostic raw data: %s' % - child_path) - - def merge(self, locator): - if isinstance(self.children, dict) and isinstance(locator.children, dict): - for k, loc in locator.children.iteritems(): - if k in self.children: - self.children[k].merge(loc) - else: - self.children[k] = loc - - def dump(self, key=None): - if key: - puts('%s "%s":%d:%d' % - (Colored.red(key), Colored.blue(self.location), self.line, self.column)) - else: - puts('"%s":%d:%d' % (Colored.blue(self.location), self.line, self.column)) - if isinstance(self.children, list): - with indent(2): - for loc in self.children: - loc.dump() - elif isinstance(self.children, dict): - with indent(2): - for k, loc in self.children.iteritems(): - loc.dump(k) - - def __str__(self): - # Should be in same format as Issue.locator_as_str - return '"%s":%d:%d' % (self.location, self.line, self.column) - - -def deepcopy_with_locators(value): - """ - Like :func:`deepcopy`, but also copies over locators. - """ - - res = deepcopy(value) - copy_locators(res, value) - return res - - -def copy_locators(target, source): - """ - Copies over ``_locator`` for all elements, recursively. - - Assumes that target and source have exactly the same list/dict structure. - """ - - locator = getattr(source, '_locator', None) - if locator is not None: - try: - setattr(target, '_locator', locator) - except AttributeError: - pass - - if isinstance(target, list) and isinstance(source, list): - for i, _ in enumerate(target): - copy_locators(target[i], source[i]) - elif isinstance(target, dict) and isinstance(source, dict): - for k, v in target.items(): - copy_locators(v, source[k]) http://git-wip-us.apache.org/repos/asf/incubator-ariatosca-website/blob/23d6ba76/apache-ariatosca-0.1.1/aria/parser/reading/raw.py ---------------------------------------------------------------------- diff --git a/apache-ariatosca-0.1.1/aria/parser/reading/raw.py b/apache-ariatosca-0.1.1/aria/parser/reading/raw.py deleted file mode 100644 index ed980ac..0000000 --- a/apache-ariatosca-0.1.1/aria/parser/reading/raw.py +++ /dev/null @@ -1,24 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from .reader import Reader - - -class RawReader(Reader): - """ - ARIA raw reader. - - Expects to receive agnostic raw data from the loader, and so does nothing to it. - """ - - def read(self): - return self.load() http://git-wip-us.apache.org/repos/asf/incubator-ariatosca-website/blob/23d6ba76/apache-ariatosca-0.1.1/aria/parser/reading/reader.py ---------------------------------------------------------------------- diff --git a/apache-ariatosca-0.1.1/aria/parser/reading/reader.py b/apache-ariatosca-0.1.1/aria/parser/reading/reader.py deleted file mode 100644 index 1a29f11..0000000 --- a/apache-ariatosca-0.1.1/aria/parser/reading/reader.py +++ /dev/null @@ -1,44 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from ...utils.openclose import OpenClose -from .exceptions import ReaderException, AlreadyReadException - - -class Reader(object): - """ - Base class for ARIA readers. - - Readers provide agnostic raw data by consuming :class:`aria.parser.loading.Loader` instances. - """ - - def __init__(self, context, location, loader): - self.context = context - self.location = location - self.loader = loader - - def load(self): - with OpenClose(self.loader) as loader: - if self.context is not None: - with self.context._locations: - for location in self.context._locations: - if location.is_equivalent(loader.location): - raise AlreadyReadException('already read: %s' % loader.location) - self.context._locations.append(loader.location) - - data = loader.load() - if data is None: - raise ReaderException('loader did not provide data: %s' % loader) - return data - - def read(self): - raise NotImplementedError http://git-wip-us.apache.org/repos/asf/incubator-ariatosca-website/blob/23d6ba76/apache-ariatosca-0.1.1/aria/parser/reading/source.py ---------------------------------------------------------------------- diff --git a/apache-ariatosca-0.1.1/aria/parser/reading/source.py b/apache-ariatosca-0.1.1/aria/parser/reading/source.py deleted file mode 100644 index 6fff2f6..0000000 --- a/apache-ariatosca-0.1.1/aria/parser/reading/source.py +++ /dev/null @@ -1,59 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from ..loading import LiteralLocation, UriLocation -from .yaml import YamlReader -from .json import JsonReader -from .jinja import JinjaReader -from .exceptions import ReaderNotFoundError - - -EXTENSIONS = { - '.yaml': YamlReader, - '.json': JsonReader, - '.jinja': JinjaReader} - - -class ReaderSource(object): - """ - Base class for ARIA reader sources. - - Reader sources provide appropriate :class:`Reader` instances for locations. - """ - - @staticmethod - def get_reader(context, location, loader): # pylint: disable=unused-argument - raise ReaderNotFoundError('location: %s' % location) - - -class DefaultReaderSource(ReaderSource): - """ - The default ARIA reader source will generate a :class:`YamlReader` for - locations that end in ".yaml", a :class:`JsonReader` for locations that - end in ".json", and a :class:`JinjaReader` for locations that end in - ".jinja". - """ - - def __init__(self, literal_reader_class=YamlReader): - super(DefaultReaderSource, self).__init__() - self.literal_reader_class = literal_reader_class - - def get_reader(self, context, location, loader): - if isinstance(location, LiteralLocation): - return self.literal_reader_class(context, location, loader) - - elif isinstance(location, UriLocation): - for extension, reader_class in EXTENSIONS.iteritems(): - if location.uri.endswith(extension): - return reader_class(context, location, loader) - - return super(DefaultReaderSource, self).get_reader(context, location, loader) http://git-wip-us.apache.org/repos/asf/incubator-ariatosca-website/blob/23d6ba76/apache-ariatosca-0.1.1/aria/parser/reading/yaml.py ---------------------------------------------------------------------- diff --git a/apache-ariatosca-0.1.1/aria/parser/reading/yaml.py b/apache-ariatosca-0.1.1/aria/parser/reading/yaml.py deleted file mode 100644 index f5eac43..0000000 --- a/apache-ariatosca-0.1.1/aria/parser/reading/yaml.py +++ /dev/null @@ -1,113 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from ruamel import yaml # @UnresolvedImport - -from ...utils.collections import OrderedDict -from .reader import Reader -from .locator import Locator -from .exceptions import ReaderSyntaxError -from .locator import LocatableString, LocatableInt, LocatableFloat - -# Add our types to ruamel.yaml -yaml.representer.RoundTripRepresenter.add_representer( - LocatableString, yaml.representer.RoundTripRepresenter.represent_unicode) -yaml.representer.RoundTripRepresenter.add_representer( - LocatableInt, yaml.representer.RoundTripRepresenter.represent_int) -yaml.representer.RoundTripRepresenter.add_representer( - LocatableFloat, yaml.representer.RoundTripRepresenter.represent_float) - -MERGE_TAG = u'tag:yaml.org,2002:merge' -MAP_TAG = u'tag:yaml.org,2002:map' - - -class YamlLocator(Locator): - """ - Map for agnostic raw data read from YAML. - """ - - def add_children(self, node): - if isinstance(node, yaml.SequenceNode): - self.children = [] - for child_node in node.value: - self.add_child(child_node) - elif isinstance(node, yaml.MappingNode): - self.children = {} - for k, child_node in node.value: - self.add_child(child_node, k) - - def add_child(self, node, key=None): - locator = YamlLocator(self.location, node.start_mark.line + 1, node.start_mark.column + 1) - if key is not None: - # Dict - if key.tag == MERGE_TAG: - for merge_key, merge_node in node.value: - self.add_child(merge_node, merge_key) - else: - self.children[key.value] = locator - else: - # List - self.children.append(locator) - locator.add_children(node) - - -def construct_yaml_map(self, node): - data = OrderedDict() - yield data - value = self.construct_mapping(node) - data.update(value) - - -yaml.constructor.SafeConstructor.add_constructor(MAP_TAG, construct_yaml_map) - - -class YamlReader(Reader): - """ - ARIA YAML reader. - """ - - def read(self): - data = self.load() - try: - data = unicode(data) - # see issue here: - # https://bitbucket.org/ruamel/yaml/issues/61/roundtriploader-causes-exceptions-with - #yaml_loader = yaml.RoundTripLoader(data) - yaml_loader = yaml.SafeLoader(data) - try: - node = yaml_loader.get_single_node() - locator = YamlLocator(self.loader.location, 0, 0) - if node is not None: - locator.add_children(node) - raw = yaml_loader.construct_document(node) - else: - raw = OrderedDict() - #locator.dump() - setattr(raw, '_locator', locator) - return raw - finally: - yaml_loader.dispose() - except yaml.parser.MarkedYAMLError as e: - context = e.context or 'while parsing' - problem = e.problem - line = e.problem_mark.line - column = e.problem_mark.column - snippet = e.problem_mark.get_snippet() - raise ReaderSyntaxError('YAML %s: %s %s' % - (e.__class__.__name__, problem, context), - location=self.loader.location, - line=line, - column=column, - snippet=snippet, - cause=e) - except Exception as e: - raise ReaderSyntaxError('YAML: %s' % e, cause=e) http://git-wip-us.apache.org/repos/asf/incubator-ariatosca-website/blob/23d6ba76/apache-ariatosca-0.1.1/aria/parser/specification.py ---------------------------------------------------------------------- diff --git a/apache-ariatosca-0.1.1/aria/parser/specification.py b/apache-ariatosca-0.1.1/aria/parser/specification.py deleted file mode 100644 index 4f452b8..0000000 --- a/apache-ariatosca-0.1.1/aria/parser/specification.py +++ /dev/null @@ -1,69 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" -Utilities for cross-referencing code with specification documents. -""" - -import re - -from ..extension import parser -from ..utils.collections import OrderedDict -from ..utils.specification import (DSL_SPECIFICATIONS, implements_specification) # pylint: disable=unused-import - - -def iter_specifications(): - """ - Iterates all specification assignments in the codebase. - """ - def iter_sections(spec, sections): - for k in sorted(sections.keys(), key=_section_key): - details = OrderedDict() - details['code'] = sections[k]['code'] - yield k, _fix_details(sections[k], spec) - - for spec, sections in DSL_SPECIFICATIONS.iteritems(): - yield spec, iter_sections(spec, sections) - - -def _section_key(value): - try: - parts = value.split('-', 1) - first = (int(v) for v in parts[0].split('.')) - second = parts[1] if len(parts) > 1 else None - return (first, second) - except ValueError: - return value - - -def _fix_details(details, spec): - code = details.get('code') - doc = details.get('doc') - url = parser.specification_url().get(spec) - - if (url is not None) and (doc is not None): - # Look for a URL in ReST docstring that begins with our url - pattern = r'<?(' - for char in url: - pattern += r'\s*' - pattern += re.escape(char) - pattern += r'[^>]+)>' - match = re.search(pattern, doc) - if match: - url = re.sub(r'\s+', '', match.group(1)) - - return OrderedDict(( - ('code', code), - ('url', url))) http://git-wip-us.apache.org/repos/asf/incubator-ariatosca-website/blob/23d6ba76/apache-ariatosca-0.1.1/aria/parser/validation/context.py ---------------------------------------------------------------------- diff --git a/apache-ariatosca-0.1.1/aria/parser/validation/context.py b/apache-ariatosca-0.1.1/aria/parser/validation/context.py deleted file mode 100644 index ef641bd..0000000 --- a/apache-ariatosca-0.1.1/aria/parser/validation/context.py +++ /dev/null @@ -1,87 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from .issue import Issue -from ...utils.threading import LockedList -from ...utils.collections import FrozenList -from ...utils.exceptions import print_exception -from ...utils.console import puts, Colored, indent -from ...utils.formatting import as_raw - - -class ValidationContext(object): - """ - Validation context. - - :ivar allow_unknown_fields: when ``False`` (the default) will report an issue if an unknown - field is used - :vartype allow_unknown_fields: bool - :ivar allow_primitive_coersion`: when ``False`` (the default) will not attempt to coerce - primitive field types - :vartype allow_primitive_coersion: bool - :ivar max_level: maximum validation level to report (default is all) - :vartype max_level: int - """ - - def __init__(self): - self.allow_unknown_fields = False - self.allow_primitive_coersion = False - self.max_level = Issue.ALL - - self._issues = LockedList() - - def report(self, message=None, exception=None, location=None, line=None, - column=None, locator=None, snippet=None, level=Issue.PLATFORM, issue=None): - if issue is None: - issue = Issue(message, exception, location, line, column, locator, snippet, level) - - # Avoid duplicate issues - with self._issues: - for i in self._issues: - if str(i) == str(issue): - return - - self._issues.append(issue) - - @property - def has_issues(self): - return len(self._issues) > 0 - - @property - def issues(self): - issues = [i for i in self._issues if i.level <= self.max_level] - issues.sort(key=lambda i: (i.level, i.location, i.line, i.column, i.message)) - return FrozenList(issues) - - @property - def issues_as_raw(self): - return [as_raw(i) for i in self.issues] - - def dump_issues(self): - issues = self.issues - if issues: - puts(Colored.blue('Validation issues:', bold=True)) - with indent(2): - for issue in issues: - puts(Colored.blue(issue.heading_as_str)) - details = issue.details_as_str - if details: - with indent(3): - puts(details) - if issue.exception is not None: - with indent(3): - print_exception(issue.exception) - return True - return False http://git-wip-us.apache.org/repos/asf/incubator-ariatosca-website/blob/23d6ba76/apache-ariatosca-0.1.1/aria/parser/validation/issue.py ---------------------------------------------------------------------- diff --git a/apache-ariatosca-0.1.1/aria/parser/validation/issue.py b/apache-ariatosca-0.1.1/aria/parser/validation/issue.py deleted file mode 100644 index db8065d..0000000 --- a/apache-ariatosca-0.1.1/aria/parser/validation/issue.py +++ /dev/null @@ -1,126 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from __future__ import absolute_import # so we can import standard 'collections' - -from ...utils.collections import OrderedDict -from ...utils.type import full_type_name - - -class Issue(object): - PLATFORM = 0 - """ - Platform error (e.g. I/O, hardware, a bug in ARIA) - """ - - SYNTAX = 1 - """ - Syntax and format (e.g. YAML, XML, JSON) - """ - - FIELD = 2 - """ - Single field - """ - - BETWEEN_FIELDS = 3 - """ - Relationships between fields within the type (internal grammar) - """ - - BETWEEN_TYPES = 4 - """ - Relationships between types (e.g. inheritance, external grammar) - """ - - BETWEEN_INSTANCES = 5 - """ - Topology (e.g. static requirements and capabilities) - """ - - EXTERNAL = 6 - """ - External (e.g. live requirements and capabilities) - """ - - ALL = 100 - - def __init__(self, message=None, exception=None, location=None, line=None, - column=None, locator=None, snippet=None, level=0): - if message is not None: - self.message = str(message) - elif exception is not None: - self.message = str(exception) - else: - self.message = 'unknown issue' - - self.exception = exception - - if locator is not None: - self.location = locator.location - self.line = locator.line - self.column = locator.column - else: - self.location = location - self.line = line - self.column = column - - self.snippet = snippet - self.level = level - - @property - def as_raw(self): - return OrderedDict(( - ('level', self.level), - ('message', self.message), - ('location', self.location), - ('line', self.line), - ('column', self.column), - ('snippet', self.snippet), - ('exception', full_type_name(self.exception) if self.exception else None))) - - @property - def locator_as_str(self): - if self.location is not None: - if self.line is not None: - if self.column is not None: - return '"%s":%d:%d' % (self.location, self.line, self.column) - else: - return '"%s":%d' % (self.location, self.line) - else: - return '"%s"' % self.location - else: - return None - - @property - def heading_as_str(self): - return '%d: %s' % (self.level, self.message) - - @property - def details_as_str(self): - details_str = '' - locator = self.locator_as_str - if locator is not None: - details_str += '@%s' % locator - if self.snippet is not None: - details_str += '\n%s' % self.snippet - return details_str - - def __str__(self): - heading_str = self.heading_as_str - details = self.details_as_str - if details: - heading_str += ', ' + details - return heading_str http://git-wip-us.apache.org/repos/asf/incubator-ariatosca-website/blob/23d6ba76/apache-ariatosca-0.1.1/aria/storage/api.py ---------------------------------------------------------------------- diff --git a/apache-ariatosca-0.1.1/aria/storage/api.py b/apache-ariatosca-0.1.1/aria/storage/api.py deleted file mode 100644 index a337743..0000000 --- a/apache-ariatosca-0.1.1/aria/storage/api.py +++ /dev/null @@ -1,186 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" -Storage APIs. -""" - -import threading - - -class StorageAPI(object): - """ - Base class for storage APIs. - """ - def create(self, **kwargs): - """ - Create a storage API. - :param kwargs: - :return: - """ - raise NotImplementedError('Subclass must implement abstract create method') - - -class ModelAPI(StorageAPI): - """ - Base class for model APIs ("MAPI"). - """ - def __init__(self, model_cls, name=None, **kwargs): - """ - :param model_cls: representing class of the model - :param name: name of the model - """ - super(ModelAPI, self).__init__(**kwargs) - self._model_cls = model_cls - self._name = name or model_cls.__modelname__ - self._thread_local = threading.local() - self._thread_local._instrumentation = [] - - @property - def _instrumentation(self): - if not hasattr(self._thread_local, '_instrumentation'): - self._thread_local._instrumentation = [] - return self._thread_local._instrumentation - - - @property - def name(self): - """ - Name of the class. - - :type: :obj:`basestring` - """ - return self._name - - @property - def model_cls(self): - """ - Class representing the model - - :type: :obj:`Type` - """ - return self._model_cls - - def get(self, entry_id, filters=None, **kwargs): - """ - Gets a model from storage. - - :param entry_id: - """ - raise NotImplementedError('Subclass must implement abstract get method') - - def put(self, entry, **kwargs): - """ - Puts a model in storage. - - :param entry: - """ - raise NotImplementedError('Subclass must implement abstract store method') - - def delete(self, entry_id, **kwargs): - """ - Deletes a model from storage. - - :param entry_id: - """ - raise NotImplementedError('Subclass must implement abstract delete method') - - def __iter__(self): - return self.iter() - - def iter(self, **kwargs): - """ - Iterate over all models in storage. - """ - raise NotImplementedError('Subclass must implement abstract iter method') - - def update(self, entry, **kwargs): - """ - Update a model in storage. - - :param entry: - :param kwargs: - """ - raise NotImplementedError('Subclass must implement abstract update method') - - -class ResourceAPI(StorageAPI): - """ - Base class for resource APIs ("RAPI"). - """ - def __init__(self, name, **kwargs): - """ - :param name: resource type - """ - super(ResourceAPI, self).__init__(**kwargs) - self._name = name - - @property - def name(self): - """ - Name of resource. - - :type: :obj:`basestring` - """ - return self._name - - def read(self, entry_id, path, **kwargs): - """ - Get a bytesteam for a resource from storage. - - :param entry_id: - :param path: - """ - raise NotImplementedError('Subclass must implement abstract read method') - - def delete(self, entry_id, path, **kwargs): - """ - Delete a resource from storage. - - :param entry_id: - :param path: - """ - raise NotImplementedError('Subclass must implement abstract delete method') - - def download(self, entry_id, destination, path=None, **kwargs): - """ - Download a resource from storage. - - :param entry_id: - :param destination: - :param path: - """ - raise NotImplementedError('Subclass must implement abstract download method') - - def upload(self, entry_id, source, path=None, **kwargs): - """ - Upload a resource to storage. - - :param entry_id: - :param source: - :param path: - """ - raise NotImplementedError('Subclass must implement abstract upload method') - - -def generate_lower_name(model_cls): - """ - Generates the name of the class from the class object, e.g. ``SomeClass`` -> ``some_class`` - - :param model_cls: class to evaluate - :return: lowercase name - :rtype: basestring - """ - return getattr(model_cls, '__mapiname__', model_cls.__tablename__) http://git-wip-us.apache.org/repos/asf/incubator-ariatosca-website/blob/23d6ba76/apache-ariatosca-0.1.1/aria/storage/collection_instrumentation.py ---------------------------------------------------------------------- diff --git a/apache-ariatosca-0.1.1/aria/storage/collection_instrumentation.py b/apache-ariatosca-0.1.1/aria/storage/collection_instrumentation.py deleted file mode 100644 index c90cb18..0000000 --- a/apache-ariatosca-0.1.1/aria/storage/collection_instrumentation.py +++ /dev/null @@ -1,314 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" -Utilities for instrumenting collections of models in storage. -""" - -from . import exceptions - - -class _InstrumentedCollection(object): - - def __init__(self, - mapi, - parent, - field_name, - field_cls, - seq=None, - is_top_level=True, - **kwargs): - self._mapi = mapi - self._parent = parent - self._field_name = field_name - self._is_top_level = is_top_level - self._field_cls = field_cls - self._load(seq, **kwargs) - - @property - def _raw(self): - raise NotImplementedError - - def _load(self, seq, **kwargs): - """ - Instantiates the object from existing seq. - - :param seq: the original sequence to load from - """ - raise NotImplementedError - - def _set(self, key, value): - """ - Sets the changes for the current object (not in the database). - - :param key: - :param value: - """ - raise NotImplementedError - - def _del(self, collection, key): - raise NotImplementedError - - def _instrument(self, key, value): - """ - Instruments any collection to track changes (and ease of access). - - :param key: - :param value: - """ - if isinstance(value, _InstrumentedCollection): - return value - elif isinstance(value, dict): - instrumentation_cls = _InstrumentedDict - elif isinstance(value, list): - instrumentation_cls = _InstrumentedList - else: - return value - - return instrumentation_cls(self._mapi, self, key, self._field_cls, value, False) - - def _raw_value(self, value): - """ - Gets the raw value. - - :param value: - """ - if isinstance(value, self._field_cls): - return value.value - return value - - def _encapsulate_value(self, key, value): - """ - Creates a new item class if needed. - - :param key: - :param value: - """ - if isinstance(value, self._field_cls): - return value - # If it is not wrapped - return self._field_cls.wrap(key, value) - - def __setitem__(self, key, value): - """ - Updates the values in both the local and the database locations. - - :param key: - :param value: - """ - self._set(key, value) - if self._is_top_level: - # We are at the top level - field = getattr(self._parent, self._field_name) - self._set_field( - field, key, value if key in field else self._encapsulate_value(key, value)) - self._mapi.update(self._parent) - else: - # We are not at the top level - self._set_field(self._parent, self._field_name, self) - - def _set_field(self, collection, key, value): - """ - Enables updating the current change in the ancestors. - - :param collection: collection to change - :param key: key for the specific field - :param value: new value - """ - if isinstance(value, _InstrumentedCollection): - value = value._raw - if key in collection and isinstance(collection[key], self._field_cls): - if isinstance(collection[key], _InstrumentedCollection): - self._del(collection, key) - collection[key].value = value - else: - collection[key] = value - return collection[key] - - def __deepcopy__(self, *args, **kwargs): - return self._raw - - -class _InstrumentedDict(_InstrumentedCollection, dict): - - def _load(self, dict_=None, **kwargs): - dict.__init__( - self, - tuple((key, self._raw_value(value)) for key, value in (dict_ or {}).items()), - **kwargs) - - def update(self, dict_=None, **kwargs): - dict_ = dict_ or {} - for key, value in dict_.items(): - self[key] = value - for key, value in kwargs.items(): - self[key] = value - - def __getitem__(self, key): - return self._instrument(key, dict.__getitem__(self, key)) - - def _set(self, key, value): - dict.__setitem__(self, key, self._raw_value(value)) - - @property - def _raw(self): - return dict(self) - - def _del(self, collection, key): - del collection[key] - - -class _InstrumentedList(_InstrumentedCollection, list): - - def _load(self, list_=None, **kwargs): - list.__init__(self, list(item for item in list_ or [])) - - def append(self, value): - self.insert(len(self), value) - - def insert(self, index, value): - list.insert(self, index, self._raw_value(value)) - if self._is_top_level: - field = getattr(self._parent, self._field_name) - field.insert(index, self._encapsulate_value(index, value)) - else: - self._parent[self._field_name] = self - - def __getitem__(self, key): - return self._instrument(key, list.__getitem__(self, key)) - - def _set(self, key, value): - list.__setitem__(self, key, value) - - def _del(self, collection, key): - del collection[key] - - @property - def _raw(self): - return list(self) - - -class _WrappedBase(object): - - def __init__(self, wrapped, instrumentation): - self._wrapped = wrapped - self._instrumentation = instrumentation - - -class _InstrumentedModel(_WrappedBase): - - def __init__(self, mapi, *args, **kwargs): - """ - The original model. - - :param wrapped: model to be instrumented - :param mapi: MAPI for the wrapped model - """ - super(_InstrumentedModel, self).__init__(*args, **kwargs) - self._mapi = mapi - self._apply_instrumentation() - - def __getattr__(self, item): - return_value = getattr(self._wrapped, item) - if isinstance(return_value, self._wrapped.__class__): - return _create_instrumented_model(return_value, self._mapi, self._instrumentation) - if isinstance(return_value, (list, dict)): - return _create_wrapped_model(return_value, self._mapi, self._instrumentation) - return return_value - - def _apply_instrumentation(self): - for field in self._instrumentation: - field_name = field.key - field_cls = field.mapper.class_ - field = getattr(self._wrapped, field_name) - - # Preserve the original value. e.g. original attributes would be located under - # _attributes - setattr(self, '_{0}'.format(field_name), field) - - # set instrumented value - if isinstance(field, dict): - instrumentation_cls = _InstrumentedDict - elif isinstance(field, list): - instrumentation_cls = _InstrumentedList - else: - # TODO: raise proper error - raise exceptions.StorageError( - "ARIA supports instrumentation for dict and list. Field {field} of the " - "class {model} is of {type} type.".format( - field=field, - model=self._wrapped, - type=type(field))) - - instrumented_class = instrumentation_cls(seq=field, - parent=self._wrapped, - mapi=self._mapi, - field_name=field_name, - field_cls=field_cls) - setattr(self, field_name, instrumented_class) - - -class _WrappedModel(_WrappedBase): - - def __init__(self, instrumentation_kwargs, *args, **kwargs): - """ - :param instrumented_cls: class to be instrumented - :param instrumentation_cls: instrumentation cls - :param wrapped: currently wrapped instance - :param kwargs: passed to the instrumented class - """ - super(_WrappedModel, self).__init__(*args, **kwargs) - self._kwargs = instrumentation_kwargs - - def _wrap(self, value): - if value.__class__ in (class_.class_ for class_ in self._instrumentation): - return _create_instrumented_model( - value, instrumentation=self._instrumentation, **self._kwargs) - elif hasattr(value, 'metadata') or isinstance(value, (dict, list)): - # Basically checks that the value is indeed an sqlmodel (it should have metadata) - return _create_wrapped_model( - value, instrumentation=self._instrumentation, **self._kwargs) - return value - - def __getattr__(self, item): - if hasattr(self, '_wrapped'): - return self._wrap(getattr(self._wrapped, item)) - else: - super(_WrappedModel, self).__getattribute__(item) - - def __getitem__(self, item): - return self._wrap(self._wrapped[item]) - - -def _create_instrumented_model(original_model, mapi, instrumentation): - return type('Instrumented{0}'.format(original_model.__class__.__name__), - (_InstrumentedModel,), - {})(wrapped=original_model, instrumentation=instrumentation, mapi=mapi) - - -def _create_wrapped_model(original_model, mapi, instrumentation): - return type('Wrapped{0}'.format(original_model.__class__.__name__), - (_WrappedModel, ), - {})(wrapped=original_model, - instrumentation=instrumentation, - instrumentation_kwargs=dict(mapi=mapi)) - - -def instrument(instrumentation, original_model, mapi): - for instrumented_field in instrumentation: - if isinstance(original_model, instrumented_field.class_): - return _create_instrumented_model(original_model, mapi, instrumentation) - - return _create_wrapped_model(original_model, mapi, instrumentation) http://git-wip-us.apache.org/repos/asf/incubator-ariatosca-website/blob/23d6ba76/apache-ariatosca-0.1.1/aria/storage/core.py ---------------------------------------------------------------------- diff --git a/apache-ariatosca-0.1.1/aria/storage/core.py b/apache-ariatosca-0.1.1/aria/storage/core.py deleted file mode 100644 index 74b1147..0000000 --- a/apache-ariatosca-0.1.1/aria/storage/core.py +++ /dev/null @@ -1,160 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" -Storage API management. -""" - -import copy -from contextlib import contextmanager - -from aria.logger import LoggerMixin -from . import sql_mapi - -__all__ = ( - 'Storage', - 'ModelStorage', - 'ResourceStorage' -) - - -class Storage(LoggerMixin): - """ - Base class for storage managers. - """ - def __init__(self, - api_cls, - api_kwargs=None, - items=(), - initiator=None, - initiator_kwargs=None, - **kwargs): - """ - :param api_cls: API class for each entry - :param api_kwargs: - :param items: items to register - :param initiator: function which initializes the storage before the first use; this function - should return a dict, this dict would be passed in addition to the API kwargs; this enables - the creation of non-serializable objects - :param initiator_kwargs: - :param kwargs: - """ - super(Storage, self).__init__(**kwargs) - self.api = api_cls - self.registered = {} - self._initiator = initiator - self._initiator_kwargs = initiator_kwargs or {} - self._api_kwargs = api_kwargs or {} - self._additional_api_kwargs = {} - if self._initiator: - self._additional_api_kwargs = self._initiator(**self._initiator_kwargs) - for item in items: - self.register(item) - self.logger.debug('{name} object is ready: {0!r}'.format( - self, name=self.__class__.__name__)) - - @property - def _all_api_kwargs(self): - kwargs = self._api_kwargs.copy() - kwargs.update(self._additional_api_kwargs) - return kwargs - - def __repr__(self): - return '{name}(api={self.api})'.format(name=self.__class__.__name__, self=self) - - def __getattr__(self, item): - try: - return self.registered[item] - except KeyError: - return super(Storage, self).__getattribute__(item) - - @property - def serialization_dict(self): - return { - 'api': self.api, - 'api_kwargs': self._api_kwargs, - 'initiator': self._initiator, - 'initiator_kwargs': self._initiator_kwargs - } - - def register(self, entry): - """ - Register an API. - - :param entry: - """ - raise NotImplementedError('Subclass must implement abstract register method') - - -class ResourceStorage(Storage): - """ - Manages storage resource APIs ("RAPIs"). - """ - def register(self, name): - """ - Register a storage resource API ("RAPI"). - - :param name: name - """ - self.registered[name] = self.api(name=name, **self._all_api_kwargs) - self.registered[name].create() - self.logger.debug('setup {name} in storage {self!r}'.format(name=name, self=self)) - - -class ModelStorage(Storage): - """ - Manages storage model APIs ("MAPIs"). - """ - def __init__(self, *args, **kwargs): - if kwargs.get('initiator', None) is None: - kwargs['initiator'] = sql_mapi.init_storage - super(ModelStorage, self).__init__(*args, **kwargs) - - def register(self, model_cls): - """ - Register a storage model API ("MAPI"). - - :param model_cls: model API to register - """ - model_name = model_cls.__modelname__ - if model_name in self.registered: - self.logger.debug('{name} in already storage {self!r}'.format(name=model_name, - self=self)) - return - self.registered[model_name] = self.api(name=model_name, - model_cls=model_cls, - **self._all_api_kwargs) - self.registered[model_name].create() - self.logger.debug('setup {name} in storage {self!r}'.format(name=model_name, self=self)) - - def drop(self): - """ - Drop all the tables. - """ - for mapi in self.registered.values(): - mapi.drop() - - @contextmanager - def instrument(self, *instrumentation): - original_instrumentation = {} - - try: - for mapi in self.registered.values(): - original_instrumentation[mapi] = copy.copy(mapi._instrumentation) - mapi._instrumentation.extend(instrumentation) - yield self - finally: - for mapi in self.registered.values(): - mapi._instrumentation[:] = original_instrumentation[mapi] http://git-wip-us.apache.org/repos/asf/incubator-ariatosca-website/blob/23d6ba76/apache-ariatosca-0.1.1/aria/storage/exceptions.py ---------------------------------------------------------------------- diff --git a/apache-ariatosca-0.1.1/aria/storage/exceptions.py b/apache-ariatosca-0.1.1/aria/storage/exceptions.py deleted file mode 100644 index c538876..0000000 --- a/apache-ariatosca-0.1.1/aria/storage/exceptions.py +++ /dev/null @@ -1,31 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" -Storage exceptions. -""" - -from .. import exceptions - - -class StorageError(exceptions.AriaError): - """ - General storage exception - """ - pass - - -class NotFoundError(StorageError): - pass http://git-wip-us.apache.org/repos/asf/incubator-ariatosca-website/blob/23d6ba76/apache-ariatosca-0.1.1/aria/storage/filesystem_rapi.py ---------------------------------------------------------------------- diff --git a/apache-ariatosca-0.1.1/aria/storage/filesystem_rapi.py b/apache-ariatosca-0.1.1/aria/storage/filesystem_rapi.py deleted file mode 100644 index b425fa2..0000000 --- a/apache-ariatosca-0.1.1/aria/storage/filesystem_rapi.py +++ /dev/null @@ -1,165 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" -File system implementation of the storage resource API ("RAPI"). -""" - -import os -import shutil -from multiprocessing import RLock -from contextlib import contextmanager -from functools import partial -from distutils import dir_util # https://github.com/PyCQA/pylint/issues/73; pylint: disable=no-name-in-module - -from aria.storage import ( - api, - exceptions -) - - -class FileSystemResourceAPI(api.ResourceAPI): - """ - File system implementation of the storage resource API ("RAPI"). - """ - - def __init__(self, directory, **kwargs): - """ - :param directory: root dir for storage - """ - super(FileSystemResourceAPI, self).__init__(**kwargs) - self.directory = directory - self.base_path = os.path.join(self.directory, self.name) - self._join_path = partial(os.path.join, self.base_path) - self._lock = RLock() - - @contextmanager - def connect(self): - """ - Establishes a connection and destroys it after use. - """ - try: - self._establish_connection() - yield self - except BaseException as e: - raise exceptions.StorageError(str(e)) - finally: - self._destroy_connection() - - def _establish_connection(self): - """ - Establishes a connection. Used in the ``connect`` context manager. - """ - self._lock.acquire() - - def _destroy_connection(self): - """ - Destroys a connection. Used in the ``connect`` context manager. - """ - self._lock.release() - - def __repr__(self): - return '{cls.__name__}(directory={self.directory})'.format( - cls=self.__class__, self=self) - - def create(self, **kwargs): - """ - Creates a directory in by path. Tries to create the root directory as well. - - :param name: path of directory - """ - try: - os.makedirs(self.directory) - except (OSError, IOError): - pass - try: - os.makedirs(self.base_path) - except (OSError, IOError): - pass - - def read(self, entry_id, path, **_): - """ - Retrieves the contents of a file. - - :param entry_id: entry ID - :param path: path to resource - :return: contents of the file - :rtype: bytes - """ - resource_relative_path = os.path.join(self.name, entry_id, path or '') - resource = os.path.join(self.directory, resource_relative_path) - if not os.path.exists(resource): - raise exceptions.StorageError("Resource {0} does not exist". - format(resource_relative_path)) - if not os.path.isfile(resource): - resources = os.listdir(resource) - if len(resources) != 1: - raise exceptions.StorageError( - 'Failed to read {0}; Reading a directory is ' - 'only allowed when it contains a single resource'.format(resource)) - resource = os.path.join(resource, resources[0]) - with open(resource, 'rb') as resource_file: - return resource_file.read() - - def download(self, entry_id, destination, path=None, **_): - """ - Downloads a file or directory. - - :param entry_id: entry ID - :param destination: download destination - :param path: path to download relative to the root of the entry (otherwise all) - """ - resource_relative_path = os.path.join(self.name, entry_id, path or '') - resource = os.path.join(self.directory, resource_relative_path) - if not os.path.exists(resource): - raise exceptions.StorageError("Resource {0} does not exist". - format(resource_relative_path)) - if os.path.isfile(resource): - shutil.copy2(resource, destination) - else: - dir_util.copy_tree(resource, destination) # pylint: disable=no-member - - def upload(self, entry_id, source, path=None, **_): - """ - Uploads a file or directory. - - :param entry_id: entry ID - :param source: source of the files to upload - :param path: the destination of the file/s relative to the entry root dir. - """ - resource_directory = os.path.join(self.directory, self.name, entry_id) - if not os.path.exists(resource_directory): - os.makedirs(resource_directory) - destination = os.path.join(resource_directory, path or '') - if os.path.isfile(source): - shutil.copy2(source, destination) - else: - dir_util.copy_tree(source, destination) # pylint: disable=no-member - - def delete(self, entry_id, path=None, **_): - """ - Deletes a file or directory. - - :param entry_id: entry ID - :param path: path to delete relative to the root of the entry (otherwise all) - """ - destination = os.path.join(self.directory, self.name, entry_id, path or '') - if os.path.exists(destination): - if os.path.isfile(destination): - os.remove(destination) - else: - shutil.rmtree(destination) - return True - return False http://git-wip-us.apache.org/repos/asf/incubator-ariatosca-website/blob/23d6ba76/apache-ariatosca-0.1.1/aria/storage/sql_mapi.py ---------------------------------------------------------------------- diff --git a/apache-ariatosca-0.1.1/aria/storage/sql_mapi.py b/apache-ariatosca-0.1.1/aria/storage/sql_mapi.py deleted file mode 100644 index 975ada7..0000000 --- a/apache-ariatosca-0.1.1/aria/storage/sql_mapi.py +++ /dev/null @@ -1,439 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" -SQLAlchemy implementation of the storage model API ("MAPI"). -""" - -import os -import platform - -from sqlalchemy import ( - create_engine, - orm, -) -from sqlalchemy.exc import SQLAlchemyError -from sqlalchemy.orm.exc import StaleDataError - -from aria.utils.collections import OrderedDict -from . import ( - api, - exceptions, - collection_instrumentation -) - -_predicates = {'ge': '__ge__', - 'gt': '__gt__', - 'lt': '__lt__', - 'le': '__le__', - 'eq': '__eq__', - 'ne': '__ne__'} - - -class SQLAlchemyModelAPI(api.ModelAPI): - """ - SQLAlchemy implementation of the storage model API ("MAPI"). - """ - - def __init__(self, - engine, - session, - **kwargs): - super(SQLAlchemyModelAPI, self).__init__(**kwargs) - self._engine = engine - self._session = session - - def get(self, entry_id, include=None, **kwargs): - """ - Returns a single result based on the model class and element ID - """ - query = self._get_query(include, {'id': entry_id}) - result = query.first() - - if not result: - raise exceptions.NotFoundError( - 'Requested `{0}` with ID `{1}` was not found' - .format(self.model_cls.__name__, entry_id) - ) - return self._instrument(result) - - def get_by_name(self, entry_name, include=None, **kwargs): - assert hasattr(self.model_cls, 'name') - result = self.list(include=include, filters={'name': entry_name}) - if not result: - raise exceptions.NotFoundError( - 'Requested {0} with name `{1}` was not found' - .format(self.model_cls.__name__, entry_name) - ) - elif len(result) > 1: - raise exceptions.StorageError( - 'Requested {0} with name `{1}` returned more than 1 value' - .format(self.model_cls.__name__, entry_name) - ) - else: - return result[0] - - def list(self, - include=None, - filters=None, - pagination=None, - sort=None, - **kwargs): - query = self._get_query(include, filters, sort) - - results, total, size, offset = self._paginate(query, pagination) - - return ListResult( - dict(total=total, size=size, offset=offset), - [self._instrument(result) for result in results] - ) - - def iter(self, - include=None, - filters=None, - sort=None, - **kwargs): - """ - Returns a (possibly empty) list of ``model_class`` results. - """ - for result in self._get_query(include, filters, sort): - yield self._instrument(result) - - def put(self, entry, **kwargs): - """ - Creatse a ``model_class`` instance from a serializable ``model`` object. - - :param entry: dict with relevant kwargs, or an instance of a class that has a ``to_dict`` - method, and whose attributes match the columns of ``model_class`` (might also be just an - instance of ``model_class``) - :return: an instance of ``model_class`` - """ - self._session.add(entry) - self._safe_commit() - return entry - - def delete(self, entry, **kwargs): - """ - Deletes a single result based on the model class and element ID. - """ - self._load_relationships(entry) - self._session.delete(entry) - self._safe_commit() - return entry - - def update(self, entry, **kwargs): - """ - Adds ``instance`` to the database session, and attempts to commit. - - :return: updated instance - """ - return self.put(entry) - - def refresh(self, entry): - """ - Reloads the instance with fresh information from the database. - - :param entry: instance to be re-loaded from the database - :return: refreshed instance - """ - self._session.refresh(entry) - self._load_relationships(entry) - return entry - - def _destroy_connection(self): - pass - - def _establish_connection(self): - pass - - def create(self, checkfirst=True, create_all=True, **kwargs): - self.model_cls.__table__.create(self._engine, checkfirst=checkfirst) - - if create_all: - # In order to create any models created dynamically (e.g. many-to-many helper tables are - # created at runtime). - self.model_cls.metadata.create_all(bind=self._engine, checkfirst=checkfirst) - - def drop(self): - """ - Drops the table. - """ - self.model_cls.__table__.drop(self._engine) - - def _safe_commit(self): - """ - Try to commit changes in the session. Roll back if exception raised SQLAlchemy errors and - rolls back if they're caught. - """ - try: - self._session.commit() - except StaleDataError as e: - self._session.rollback() - raise exceptions.StorageError('Version conflict: {0}'.format(str(e))) - except (SQLAlchemyError, ValueError) as e: - self._session.rollback() - raise exceptions.StorageError('SQL Storage error: {0}'.format(str(e))) - - def _get_base_query(self, include, joins): - """ - Create the initial query from the model class and included columns. - - :param include: (possibly empty) list of columns to include in the query - :return: SQLAlchemy AppenderQuery object - """ - # If only some columns are included, query through the session object - if include: - # Make sure that attributes come before association proxies - include.sort(key=lambda x: x.is_clause_element) - query = self._session.query(*include) - else: - # If all columns should be returned, query directly from the model - query = self._session.query(self.model_cls) - - query = query.join(*joins) - return query - - @staticmethod - def _get_joins(model_class, columns): - """ - Gets a list of all the tables on which we need to join. - - :param columns: set of all attributes involved in the query - """ - - # Using a list instead of a set because order is important - joins = OrderedDict() - for column_name in columns: - column = getattr(model_class, column_name) - while not column.is_attribute: - join_attr = column.local_attr - # This is a hack, to deal with the fact that SQLA doesn't - # fully support doing something like: `if join_attr in joins`, - # because some SQLA elements have their own comparators - join_attr_name = str(join_attr) - if join_attr_name not in joins: - joins[join_attr_name] = join_attr - column = column.remote_attr - - return joins.values() - - @staticmethod - def _sort_query(query, sort=None): - """ - Adds sorting clauses to the query. - - :param query: base SQL query - :param sort: optional dictionary where keys are column names to sort by, and values are - the order (asc/desc) - :return: SQLAlchemy AppenderQuery object - """ - if sort: - for column, order in sort.items(): - if order == 'desc': - column = column.desc() - query = query.order_by(column) - return query - - def _filter_query(self, query, filters): - """ - Adds filter clauses to the query. - - :param query: base SQL query - :param filters: optional dictionary where keys are column names to filter by, and values - are values applicable for those columns (or lists of such values) - :return: SQLAlchemy AppenderQuery object - """ - return self._add_value_filter(query, filters) - - @staticmethod - def _add_value_filter(query, filters): - for column, value in filters.items(): - if isinstance(value, dict): - for predicate, operand in value.items(): - query = query.filter(getattr(column, predicate)(operand)) - elif isinstance(value, (list, tuple)): - query = query.filter(column.in_(value)) - else: - query = query.filter(column == value) - - return query - - def _get_query(self, - include=None, - filters=None, - sort=None): - """ - Gets a SQL query object based on the params passed. - - :param model_class: SQL database table class - :param include: optional list of columns to include in the query - :param filters: optional dictionary where keys are column names to filter by, and values - are values applicable for those columns (or lists of such values) - :param sort: optional dictionary where keys are column names to sort by, and values are the - order (asc/desc) - :return: sorted and filtered query with only the relevant columns - """ - include, filters, sort, joins = self._get_joins_and_converted_columns( - include, filters, sort - ) - filters = self._convert_operands(filters) - - query = self._get_base_query(include, joins) - query = self._filter_query(query, filters) - query = self._sort_query(query, sort) - return query - - @staticmethod - def _convert_operands(filters): - for column, conditions in filters.items(): - if isinstance(conditions, dict): - for predicate, operand in conditions.items(): - if predicate not in _predicates: - raise exceptions.StorageError( - "{0} is not a valid predicate for filtering. Valid predicates are {1}" - .format(predicate, ', '.join(_predicates.keys()))) - del filters[column][predicate] - filters[column][_predicates[predicate]] = operand - - - return filters - - def _get_joins_and_converted_columns(self, - include, - filters, - sort): - """ - Gets a list of tables on which we need to join and the converted ``include``, ``filters`` - and ```sort`` arguments (converted to actual SQLAlchemy column/label objects instead of - column names). - """ - include = include or [] - filters = filters or dict() - sort = sort or OrderedDict() - - all_columns = set(include) | set(filters.keys()) | set(sort.keys()) - joins = self._get_joins(self.model_cls, all_columns) - - include, filters, sort = self._get_columns_from_field_names( - include, filters, sort - ) - return include, filters, sort, joins - - def _get_columns_from_field_names(self, - include, - filters, - sort): - """ - Gooes over the optional parameters (include, filters, sort), and replace column names with - actual SQLAlechmy column objects. - """ - include = [self._get_column(c) for c in include] - filters = dict((self._get_column(c), filters[c]) for c in filters) - sort = OrderedDict((self._get_column(c), sort[c]) for c in sort) - - return include, filters, sort - - def _get_column(self, column_name): - """ - Returns the column on which an action (filtering, sorting, etc.) would need to be performed. - Can be either an attribute of the class, or an association proxy linked to a relationship - in the class. - """ - column = getattr(self.model_cls, column_name) - if column.is_attribute: - return column - else: - # We need to get to the underlying attribute, so we move on to the - # next remote_attr until we reach one - while not column.remote_attr.is_attribute: - column = column.remote_attr - # Put a label on the remote attribute with the name of the column - return column.remote_attr.label(column_name) - - @staticmethod - def _paginate(query, pagination): - """ - Paginates the query by size and offset. - - :param query: current SQLAlchemy query object - :param pagination: optional dict with size and offset keys - :return: tuple with four elements: - * results: ``size`` items starting from ``offset`` - * the total count of items - * ``size`` [default: 0] - * ``offset`` [default: 0] - """ - if pagination: - size = pagination.get('size', 0) - offset = pagination.get('offset', 0) - total = query.order_by(None).count() # Fastest way to count - results = query.limit(size).offset(offset).all() - return results, total, size, offset - else: - results = query.all() - return results, len(results), 0, 0 - - @staticmethod - def _load_relationships(instance): - """ - Helper method used to overcome a problem where the relationships that rely on joins aren't - being loaded automatically. - """ - for rel in instance.__mapper__.relationships: - getattr(instance, rel.key) - - def _instrument(self, model): - if self._instrumentation: - return collection_instrumentation.instrument(self._instrumentation, model, self) - else: - return model - - -def init_storage(base_dir, filename='db.sqlite'): - """ - Built-in ModelStorage initiator. - - Creates a SQLAlchemy engine and a session to be passed to the MAPI. - - ``initiator_kwargs`` must be passed to the ModelStorage which must hold the ``base_dir`` for the - location of the database file, and an option filename. This would create an SQLite database. - - :param base_dir: directory of the database - :param filename: database file name. - :return: - """ - uri = 'sqlite:///{platform_char}{path}'.format( - # Handles the windows behavior where there is not root, but drivers. - # Thus behaving as relative path. - platform_char='' if 'Windows' in platform.system() else '/', - - path=os.path.join(base_dir, filename)) - - engine = create_engine(uri, connect_args=dict(timeout=15)) - - session_factory = orm.sessionmaker(bind=engine) - session = orm.scoped_session(session_factory=session_factory) - - return dict(engine=engine, session=session) - - -class ListResult(list): - """ - Contains results about the requested items. - """ - def __init__(self, metadata, *args, **qwargs): - super(ListResult, self).__init__(*args, **qwargs) - self.metadata = metadata - self.items = self http://git-wip-us.apache.org/repos/asf/incubator-ariatosca-website/blob/23d6ba76/apache-ariatosca-0.1.1/aria/utils/archive.py ---------------------------------------------------------------------- diff --git a/apache-ariatosca-0.1.1/aria/utils/archive.py b/apache-ariatosca-0.1.1/aria/utils/archive.py deleted file mode 100644 index 29efcb1..0000000 --- a/apache-ariatosca-0.1.1/aria/utils/archive.py +++ /dev/null @@ -1,66 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" -Archive utilities. -""" - -import os -import tarfile -import zipfile -import tempfile -from contextlib import closing - - -def is_archive(source): - return tarfile.is_tarfile(source) or zipfile.is_zipfile(source) - - -def extract_archive(source): - if tarfile.is_tarfile(source): - return untar(source) - elif zipfile.is_zipfile(source): - return unzip(source) - raise ValueError( - 'Unsupported archive type provided or archive is not valid: {0}.'.format(source)) - - -def tar(source, destination): - with closing(tarfile.open(destination, 'w:gz')) as tar_archive: - tar_archive.add(source, arcname=os.path.basename(source)) - - -def untar(archive, destination=None): - if not destination: - destination = tempfile.mkdtemp() - with closing(tarfile.open(name=archive)) as tar_archive: - tar_archive.extractall(path=destination, members=tar_archive.getmembers()) - return destination - - -def zip(source, destination): - with closing(zipfile.ZipFile(destination, 'w')) as zip_file: - for root, _, files in os.walk(source): - for filename in files: - file_path = os.path.join(root, filename) - source_dir = os.path.dirname(source) - zip_file.write( - file_path, os.path.relpath(file_path, source_dir)) - return destination - - -def unzip(archive, destination=None): - if not destination: - destination = tempfile.mkdtemp() - with closing(zipfile.ZipFile(archive, 'r')) as zip_file: - zip_file.extractall(destination) - return destination
