Repository: incubator-ariatosca Updated Branches: refs/heads/SQLAlchemy-based-models 512b4bc6a -> 7ba33f1d5
lynting and tests fixing Project: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/commit/7ba33f1d Tree: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/tree/7ba33f1d Diff: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/diff/7ba33f1d Branch: refs/heads/SQLAlchemy-based-models Commit: 7ba33f1d53ab0c482e0c733135a4d53bb18bb952 Parents: 512b4bc Author: mxmrlv <mxm...@gmail.com> Authored: Thu Dec 1 12:31:25 2016 +0200 Committer: mxmrlv <mxm...@gmail.com> Committed: Thu Dec 1 12:31:25 2016 +0200 ---------------------------------------------------------------------- aria/__init__.py | 30 ++++++++-------- aria/orchestrator/workflows/api/task.py | 9 +++-- aria/storage/__init__.py | 38 +++++++++++++++++--- aria/storage/api.py | 2 +- aria/storage/exceptions.py | 4 ++- aria/storage/filesystem_api.py | 13 ++----- aria/storage/mapi/filesystem.py | 4 +-- aria/storage/mapi/inmemory.py | 2 +- aria/storage/mapi/sql.py | 13 ++++--- aria/storage/models.py | 2 +- aria/storage/rapi/filesystem.py | 18 ++++++---- aria/storage/structures.py | 11 ++++-- .../orchestrator/workflows/builtin/__init__.py | 5 --- tests/storage/test_resource_storage.py | 2 +- 14 files changed, 91 insertions(+), 62 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/7ba33f1d/aria/__init__.py ---------------------------------------------------------------------- diff --git a/aria/__init__.py b/aria/__init__.py index 1fd46c7..2c00729 100644 --- a/aria/__init__.py +++ b/aria/__init__.py @@ -63,21 +63,21 @@ def application_model_storage(api, api_params=None): Initiate model storage for the supplied storage driver """ models = [ - storage.models.Blueprint, - storage.models.Deployment, - storage.models.Node, - storage.models.NodeInstance, - storage.models.Relationship, - storage.models.RelationshipInstance, - storage.models.Plugin, - storage.models.Snapshot, - storage.models.DeploymentUpdate, - storage.models.DeploymentUpdateStep, - storage.models.DeploymentModification, - storage.models.Execution, - storage.models.ProviderContext, - storage.models.Task, - ] + storage.models.Blueprint, + storage.models.Deployment, + storage.models.Node, + storage.models.NodeInstance, + storage.models.Relationship, + storage.models.RelationshipInstance, + storage.models.Plugin, + storage.models.Snapshot, + storage.models.DeploymentUpdate, + storage.models.DeploymentUpdateStep, + storage.models.DeploymentModification, + storage.models.Execution, + storage.models.ProviderContext, + storage.models.Task, + ] # if api not in _model_storage: _model_storage[api] = storage.ModelStorage(api, items=models, api_params=api_params or {}) return _model_storage[api] http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/7ba33f1d/aria/orchestrator/workflows/api/task.py ---------------------------------------------------------------------- diff --git a/aria/orchestrator/workflows/api/task.py b/aria/orchestrator/workflows/api/task.py index e95ff6d..358315c 100644 --- a/aria/orchestrator/workflows/api/task.py +++ b/aria/orchestrator/workflows/api/task.py @@ -18,7 +18,6 @@ Provides the tasks to be entered into the task graph """ from uuid import uuid4 -import aria from aria import storage from ... import context @@ -76,8 +75,8 @@ class OperationTask(BaseTask): :param actor: the operation host on which this operation is registered. :param inputs: operation inputs. """ - assert isinstance(actor, (aria.storage.models.NodeInstance, - aria.storage.models.RelationshipInstance)) + assert isinstance(actor, (storage.models.NodeInstance, + storage.models.RelationshipInstance)) super(OperationTask, self).__init__() self.actor = actor self.name = '{name}.{actor.id}'.format(name=name, actor=actor) @@ -98,7 +97,7 @@ class OperationTask(BaseTask): :param instance: the node of which this operation belongs to. :param name: the name of the operation. """ - assert isinstance(instance, aria.storage.models.NodeInstance) + assert isinstance(instance, storage.models.NodeInstance) operation_details = instance.node.operations[name] operation_inputs = operation_details.get('inputs', {}) operation_inputs.update(inputs or {}) @@ -120,7 +119,7 @@ class OperationTask(BaseTask): with 'source_operations' and 'target_operations' :param inputs any additional inputs to the operation """ - assert isinstance(instance, aria.storage.models.RelationshipInstance) + assert isinstance(instance, storage.models.RelationshipInstance) if operation_end not in [cls.TARGET_OPERATION, cls.SOURCE_OPERATION]: raise exceptions.TaskException('The operation end should be {0} or {1}'.format( cls.TARGET_OPERATION, cls.SOURCE_OPERATION http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/7ba33f1d/aria/storage/__init__.py ---------------------------------------------------------------------- diff --git a/aria/storage/__init__.py b/aria/storage/__init__.py index db5caa9..6740cd0 100644 --- a/aria/storage/__init__.py +++ b/aria/storage/__init__.py @@ -39,9 +39,12 @@ API: """ from aria.logger import LoggerMixin -from aria.storage import api -from aria.storage.exceptions import StorageError -from . import models, exceptions, api, structures +from . import ( + models, + exceptions, + api as storage_api, + structures +) __all__ = ( @@ -74,20 +77,41 @@ class Storage(LoggerMixin): except KeyError: return super(Storage, self).__getattribute__(item) - def register(self, name): + def register(self, entry): + """ + Register the entry to the storage + :param name: + :return: + """ raise NotImplementedError('Subclass must implement abstract register method') class ResourceStorage(Storage): + """ + Represents resource storage. + """ def register(self, name): + """ + Register the resource type to resource storage. + :param name: + :return: + """ self.registered[name] = self.api(name=name, **self._api_params) self.registered[name].create() self.logger.debug('setup {name} in storage {self!r}'.format(name=name, self=self)) class ModelStorage(Storage): + """ + Represents model storage. + """ def register(self, model): - model_name = api.generate_lower_name(model) + """ + Register the model into the model storage. + :param model: the model to register. + :return: + """ + model_name = storage_api.generate_lower_name(model) if model_name in self.registered: self.logger.debug('{name} in already storage {self!r}'.format(name=model_name, self=self)) @@ -97,5 +121,9 @@ class ModelStorage(Storage): self.logger.debug('setup {name} in storage {self!r}'.format(name=model_name, self=self)) def drop(self): + """ + Drop all the tables from the model. + :return: + """ for mapi in self.registered.values(): mapi.drop() http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/7ba33f1d/aria/storage/api.py ---------------------------------------------------------------------- diff --git a/aria/storage/api.py b/aria/storage/api.py index 3ca1d1b..7bdbd5d 100644 --- a/aria/storage/api.py +++ b/aria/storage/api.py @@ -99,7 +99,7 @@ class ModelAPI(StorageAPI): """ return self._model_cls - def get(self, entry_id, **kwargs): + def get(self, entry_id, filters=None, **kwargs): """ Get entry from storage. http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/7ba33f1d/aria/storage/exceptions.py ---------------------------------------------------------------------- diff --git a/aria/storage/exceptions.py b/aria/storage/exceptions.py index 22dfc50..f982f63 100644 --- a/aria/storage/exceptions.py +++ b/aria/storage/exceptions.py @@ -12,7 +12,9 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - +""" +Storage based exceptions +""" from .. import exceptions http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/7ba33f1d/aria/storage/filesystem_api.py ---------------------------------------------------------------------- diff --git a/aria/storage/filesystem_api.py b/aria/storage/filesystem_api.py index 03800ff..d42cd61 100644 --- a/aria/storage/filesystem_api.py +++ b/aria/storage/filesystem_api.py @@ -20,12 +20,12 @@ from multiprocessing import RLock from . import api -class FileSystemAPI(api.StorageAPI): +class BaseFileSystemAPI(api.StorageAPI): """ Base class which handles storage on the file system. """ def __init__(self, *args, **kwargs): - super(FileSystemAPI, self).__init__(*args, **kwargs) + super(BaseFileSystemAPI, self).__init__(*args, **kwargs) self._lock = RLock() def _establish_connection(self): @@ -33,12 +33,3 @@ class FileSystemAPI(api.StorageAPI): def _destroy_connection(self): self._lock.release() - - def __getstate__(self): - obj_dict = super(FileSystemAPI, self).__getstate__() - del obj_dict['_lock'] - return obj_dict - - def __setstate__(self, obj_dict): - super(FileSystemAPI, self).__setstate__(obj_dict) - vars(self).update(_lock=RLock(), **obj_dict) http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/7ba33f1d/aria/storage/mapi/filesystem.py ---------------------------------------------------------------------- diff --git a/aria/storage/mapi/filesystem.py b/aria/storage/mapi/filesystem.py index 2a091d7..fa24869 100644 --- a/aria/storage/mapi/filesystem.py +++ b/aria/storage/mapi/filesystem.py @@ -26,9 +26,7 @@ from .. import ( ) -# TODO: fix docs - -class FileSystemModelAPI(api.ModelAPI, filesystem_api.FileSystemAPI): +class FileSystemModelAPI(api.ModelAPI, filesystem_api.BaseFileSystemAPI): """ File system model storage. """ http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/7ba33f1d/aria/storage/mapi/inmemory.py ---------------------------------------------------------------------- diff --git a/aria/storage/mapi/inmemory.py b/aria/storage/mapi/inmemory.py index 986c43a..09dbcfc 100644 --- a/aria/storage/mapi/inmemory.py +++ b/aria/storage/mapi/inmemory.py @@ -15,7 +15,7 @@ # DEPRECATED -# NOQA +#pylint: skip-file from collections import namedtuple http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/7ba33f1d/aria/storage/mapi/sql.py ---------------------------------------------------------------------- diff --git a/aria/storage/mapi/sql.py b/aria/storage/mapi/sql.py index 1e60f87..f2ec0e5 100644 --- a/aria/storage/mapi/sql.py +++ b/aria/storage/mapi/sql.py @@ -15,12 +15,13 @@ """ SQLalchemy based MAPI """ -from collections import OrderedDict from sqlite3 import DatabaseError as SQLiteDBError from sqlalchemy.exc import SQLAlchemyError from sqlalchemy.sql.elements import Label +from aria.utils.collections import OrderedDict + try: from psycopg2 import DatabaseError as Psycopg2DBError @@ -139,6 +140,10 @@ class SQLAlchemyModelAPI(storage.api.ModelAPI): self.model_cls.__table__.create(self._engine) def drop(self): + """ + Drop the table from the storage. + :return: + """ self.model_cls.__table__.drop(self._engine) def _safe_commit(self): @@ -258,7 +263,7 @@ class SQLAlchemyModelAPI(storage.api.ModelAPI): else: include.insert(0, col) - filters = {self._get_column(c): filters[c] for c in filters} + filters = dict((self._get_column(c), filters[c]) for c in filters) sort = OrderedDict((self._get_column(c), sort[c]) for c in sort) return include, filters, sort @@ -271,8 +276,8 @@ class SQLAlchemyModelAPI(storage.api.ModelAPI): return [] all_column_names = include + filters.keys() + sort.keys() - join_columns = {column_name for column_name in all_column_names - if self._is_join_column(column_name)} + join_columns = set(column_name for column_name in all_column_names + if self._is_join_column(column_name)) # If the only columns included are the columns on which we would # normally join, there isn't actually a need to join, as the FROM http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/7ba33f1d/aria/storage/models.py ---------------------------------------------------------------------- diff --git a/aria/storage/models.py b/aria/storage/models.py index 55b9247..9d0515e 100644 --- a/aria/storage/models.py +++ b/aria/storage/models.py @@ -50,7 +50,7 @@ from .structures import ( String, PickleType, Float, - MutableDict, + MutableDict, Dict, foreign_key, one_to_many_relationship http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/7ba33f1d/aria/storage/rapi/filesystem.py ---------------------------------------------------------------------- diff --git a/aria/storage/rapi/filesystem.py b/aria/storage/rapi/filesystem.py index 7abbe07..ceafde5 100644 --- a/aria/storage/rapi/filesystem.py +++ b/aria/storage/rapi/filesystem.py @@ -20,12 +20,14 @@ import shutil from distutils import dir_util from functools import partial -from aria.storage import StorageError -from aria.storage import api -from aria.storage import filesystem_api +from aria.storage import ( + api, + filesystem_api, + exceptions +) -class FileSystemResourceAPI(api.ResourceAPI, filesystem_api.FileSystemAPI): +class FileSystemResourceAPI(api.ResourceAPI, filesystem_api.BaseFileSystemAPI): """ File system resource storage. """ @@ -69,11 +71,12 @@ class FileSystemResourceAPI(api.ResourceAPI, filesystem_api.FileSystemAPI): resource_relative_path = os.path.join(self.name, entry_id, path or '') resource = os.path.join(self.directory, resource_relative_path) if not os.path.exists(resource): - raise StorageError("Resource {0} does not exist".format(resource_relative_path)) + raise exceptions.StorageError("Resource {0} does not exist". + format(resource_relative_path)) if not os.path.isfile(resource): resources = os.listdir(resource) if len(resources) != 1: - raise StorageError('No resource in path: {0}'.format(resource)) + raise exceptions.StorageError('No resource in path: {0}'.format(resource)) resource = os.path.join(resource, resources[0]) with open(resource, 'rb') as resource_file: return resource_file.read() @@ -90,7 +93,8 @@ class FileSystemResourceAPI(api.ResourceAPI, filesystem_api.FileSystemAPI): resource_relative_path = os.path.join(self.name, entry_id, path or '') resource = os.path.join(self.directory, resource_relative_path) if not os.path.exists(resource): - raise StorageError("Resource {0} does not exist".format(resource_relative_path)) + raise exceptions.StorageError("Resource {0} does not exist". + format(resource_relative_path)) if os.path.isfile(resource): shutil.copy2(resource, destination) else: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/7ba33f1d/aria/storage/structures.py ---------------------------------------------------------------------- diff --git a/aria/storage/structures.py b/aria/storage/structures.py index 8578e77..4f01524 100644 --- a/aria/storage/structures.py +++ b/aria/storage/structures.py @@ -151,9 +151,16 @@ def many_to_many_relationship( class Dict(TypeDecorator): """ - Dict represenation of type. + Dict representation of type. """ + def process_literal_param(self, value, dialect): + pass + + @property + def python_type(self): + return dict + impl = VARCHAR def process_bind_param(self, value, dialect): @@ -224,7 +231,7 @@ class SQLModelBase(Model): Convert the model into json. :return: """ - return jsonpickle.encode(self.to_dict(), unpicklable=False) + return jsonpickle.encode(self.to_dict, unpicklable=False) @classproperty def fields(cls): http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/7ba33f1d/tests/orchestrator/workflows/builtin/__init__.py ---------------------------------------------------------------------- diff --git a/tests/orchestrator/workflows/builtin/__init__.py b/tests/orchestrator/workflows/builtin/__init__.py index 4d465a0..7649a2a 100644 --- a/tests/orchestrator/workflows/builtin/__init__.py +++ b/tests/orchestrator/workflows/builtin/__init__.py @@ -53,8 +53,3 @@ def assert_node_uninstall_operations(operations, with_relationships=False): else: for i, operation in enumerate(operations): assert operation.name.startswith(mock.operations.NODE_OPERATIONS_UNINSTALL[i]) - - -@pytest.fixture(autouse=True) -def teardown(): - from aria import storage http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/7ba33f1d/tests/storage/test_resource_storage.py ---------------------------------------------------------------------- diff --git a/tests/storage/test_resource_storage.py b/tests/storage/test_resource_storage.py index 06d0810..452867e 100644 --- a/tests/storage/test_resource_storage.py +++ b/tests/storage/test_resource_storage.py @@ -1,4 +1,4 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more +# Licensed to the Apache ftware Foundation (ASF) under one or more # contributor license agreements. See the NOTICE file distributed with # this work for additional information regarding copyright ownership. # The ASF licenses this file to You under the Apache License, Version 2.0
