Repository: incubator-ariatosca Updated Branches: refs/heads/SQLAlchemy-based-models c8f0fe26d -> 749f37555
wip Project: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/commit/749f3755 Tree: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/tree/749f3755 Diff: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/diff/749f3755 Branch: refs/heads/SQLAlchemy-based-models Commit: 749f37555f6a60e770018a7dfb5beb178ea0a937 Parents: c8f0fe2 Author: mxmrlv <mxm...@gmail.com> Authored: Wed Nov 30 19:01:57 2016 +0200 Committer: mxmrlv <mxm...@gmail.com> Committed: Wed Nov 30 19:01:57 2016 +0200 ---------------------------------------------------------------------- aria/__init__.py | 38 +-- aria/storage/__init__.py | 18 +- aria/storage/mapi/inmemory.py | 4 +- aria/storage/mapi/sql.py | 60 ++-- aria/storage/models.py | 348 +++++++++++----------- aria/storage/structures.py | 51 +--- tests/mock/context.py | 13 +- tests/orchestrator/context/test_operation.py | 4 +- tests/orchestrator/context/test_toolbelt.py | 5 +- tests/orchestrator/context/test_workflow.py | 3 +- tests/storage/__init__.py | 7 - tests/storage/test_model_storage.py | 51 ++-- 12 files changed, 285 insertions(+), 317 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/749f3755/aria/__init__.py ---------------------------------------------------------------------- diff --git a/aria/__init__.py b/aria/__init__.py index 374022b..7e7feee 100644 --- a/aria/__init__.py +++ b/aria/__init__.py @@ -37,6 +37,7 @@ __all__ = ( 'operation', ) +_model_storage = {} _resource_storage = {} @@ -61,23 +62,26 @@ def application_model_storage(api, api_params=None): """ Initiate model storage for the supplied storage driver """ - return storage.ModelStorage( - api, api_params=api_params or {}, items=[ - storage.models.Node, - storage.models.Relationship, - storage.models.NodeInstance, - storage.models.RelationshipInstance, - storage.models.Plugin, - storage.models.Snapshot, - storage.models.Blueprint, - storage.models.Deployment, - storage.models.DeploymentUpdate, - storage.models.DeploymentUpdateStep, - storage.models.DeploymentModification, - storage.models.Execution, - storage.models.ProviderContext, - storage.models.Task, - ]) + models = [ + storage.models.Blueprint, + storage.models.Deployment, + storage.models.Node, + storage.models.NodeInstance, + storage.models.Relationship, + storage.models.RelationshipInstance, + storage.models.Plugin, + storage.models.Snapshot, + storage.models.DeploymentUpdate, + storage.models.DeploymentUpdateStep, + storage.models.DeploymentModification, + storage.models.Execution, + storage.models.ProviderContext, + storage.models.Task, + ] + # if api not in _model_storage: + _model_storage[api] = storage.ModelStorage(api, api_params=api_params or {}, items=models) + return _model_storage[api] + def application_resource_storage(driver): http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/749f3755/aria/storage/__init__.py ---------------------------------------------------------------------- diff --git a/aria/storage/__init__.py b/aria/storage/__init__.py index e5b1343..db5caa9 100644 --- a/aria/storage/__init__.py +++ b/aria/storage/__init__.py @@ -42,7 +42,6 @@ from aria.logger import LoggerMixin from aria.storage import api from aria.storage.exceptions import StorageError from . import models, exceptions, api, structures -from .structures import db __all__ = ( @@ -89,13 +88,14 @@ class ResourceStorage(Storage): class ModelStorage(Storage): def register(self, model): model_name = api.generate_lower_name(model) - if model_name not in self.registered: - self.registered[model_name] = self.api(name=model_name, - model_cls=model, - **self._api_params) - self.registered[model_name].create() - self.logger.debug('setup {name} in storage {self!r}'.format(name=model_name, self=self)) - else: + if model_name in self.registered: self.logger.debug('{name} in already storage {self!r}'.format(name=model_name, self=self)) - + return + self.registered[model_name] = self.api(name=model_name, model_cls=model, **self._api_params) + self.registered[model_name].create() + self.logger.debug('setup {name} in storage {self!r}'.format(name=model_name, self=self)) + + def drop(self): + for mapi in self.registered.values(): + mapi.drop() http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/749f3755/aria/storage/mapi/inmemory.py ---------------------------------------------------------------------- diff --git a/aria/storage/mapi/inmemory.py b/aria/storage/mapi/inmemory.py index 10c730a..986c43a 100644 --- a/aria/storage/mapi/inmemory.py +++ b/aria/storage/mapi/inmemory.py @@ -19,9 +19,9 @@ from collections import namedtuple -from aria.storage.structures import db from .. import api +from ..structures import orm _Pointer = namedtuple('_Pointer', 'name, is_iter') @@ -52,7 +52,7 @@ class InMemoryModelAPI(api.ModelAPI): def _setup_pointers_mapping(self): for field_name, field_cls in vars(self.model_cls).items(): if not (getattr(field_cls, 'impl', None) is not None and - isinstance(field_cls.impl.parent_token, db.RelationshipProperty)): + isinstance(field_cls.impl.parent_token, orm.RelationshipProperty)): continue pointer_key = _Pointer(field_name, is_iter=False) self.pointer_mapping[pointer_key] = self.__class__( http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/749f3755/aria/storage/mapi/sql.py ---------------------------------------------------------------------- diff --git a/aria/storage/mapi/sql.py b/aria/storage/mapi/sql.py index 57f143d..f91d614 100644 --- a/aria/storage/mapi/sql.py +++ b/aria/storage/mapi/sql.py @@ -18,9 +18,12 @@ SQLalchemy based MAPI from collections import OrderedDict from sqlite3 import DatabaseError as SQLiteDBError +from sqlalchemy import create_engine from sqlalchemy.exc import SQLAlchemyError +from sqlalchemy.orm import scoped_session from sqlalchemy.sql.elements import Label + try: from psycopg2 import DatabaseError as Psycopg2DBError sql_errors = (SQLAlchemyError, SQLiteDBError, Psycopg2DBError) @@ -38,21 +41,18 @@ class SQLAlchemyModelAPI(storage.api.ModelAPI): """ SQL based MAPI. """ + def __init__(self, - app, - sql_dialect=DEFAULT_SQL_DIALECT, - username='', - password='', - host='localhost', - db_name='', + engine, + session, + # sql_dialect=DEFAULT_SQL_DIALECT, + # username='', + # password='', + # host='localhost', + # db_name='', **kwargs): super(SQLAlchemyModelAPI, self).__init__(**kwargs) - self._app = app - self._sql_dialect = sql_dialect - self._username = username - self._password = password - self._host = host - self._db_name = db_name + self._session = session def get(self, entry_id, include=None, filters=None, locking=False): """Return a single result based on the model class and element ID @@ -92,7 +92,7 @@ class SQLAlchemyModelAPI(storage.api.ModelAPI): of `model_class` (might also my just an instance of `model_class`) :return: An instance of `model_class` """ - storage.db.session.add(entry) + self._session.add(entry) self._safe_commit() return entry @@ -113,7 +113,7 @@ class SQLAlchemyModelAPI(storage.api.ModelAPI): ) ) self._load_properties(instance) - storage.db.session.delete(instance) + self._session.delete(instance) self._safe_commit() return instance @@ -131,7 +131,7 @@ class SQLAlchemyModelAPI(storage.api.ModelAPI): :param entry: Instance to be re-loaded from the DB :return: The refreshed instance """ - storage.db.session.refresh(entry) + self._session.refresh(entry) self._load_properties(entry) return entry @@ -142,33 +142,19 @@ class SQLAlchemyModelAPI(storage.api.ModelAPI): pass def create(self): - if self._app.config.get('SQLALCHEMY_DATABASE_URI', None) is None: - sql_uri_template = '{sql_dialect}://' - if self._sql_dialect != DEFAULT_SQL_DIALECT: - sql_uri_template = sql_uri_template + '{username}:{password}@{hostname}/{db_name}' - self._app.config['SQLALCHEMY_DATABASE_URI'] = sql_uri_template.format( - sql_dialect=self._sql_dialect, - username=self._username, - password=self._password, - hostname=self._host, - db_name=self._db_name) - - with self._app.app_context(): - if storage.structures.db.app is None: - storage.structures.db.app = self._app - storage.structures.db.init_app(self._app) - self.model_cls.__table__.create(storage.structures.db.session.bind) + self.model_cls.__table__.create(self._engine) + def drop(self): + self.model_cls.__table__.drop(self._engine) - @staticmethod - def _safe_commit(): + def _safe_commit(self): """Try to commit changes in the session. Roll back if exception raised Excepts SQLAlchemy errors and rollbacks if they're caught """ try: - storage.db.session.commit() + self._session.commit() except sql_errors as e: - storage.db.session.rollback() + self._session.rollback() raise storage.exceptions.StorageError( 'SQL Storage error: {0}'.format(str(e)) ) @@ -185,10 +171,10 @@ class SQLAlchemyModelAPI(storage.api.ModelAPI): # If only some columns are included, query through the session object if include: - query = storage.db.session.query(*include) + query = self._session.query(*include) else: # If all columns should be returned, query directly from the model - query = self.model_cls.query + query = self._session.query(self.model_cls) # Add any joins that might be necessary for join_model in joins: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/749f3755/aria/storage/models.py ---------------------------------------------------------------------- diff --git a/aria/storage/models.py b/aria/storage/models.py index 5a8d017..55b9247 100644 --- a/aria/storage/models.py +++ b/aria/storage/models.py @@ -39,9 +39,9 @@ classes: from datetime import datetime from uuid import uuid4 -from aria.storage import structures from .structures import ( - schema, + SQLModelBase, + Column, Integer, Text, DateTime, @@ -50,6 +50,10 @@ from .structures import ( String, PickleType, Float, + MutableDict, + Dict, + foreign_key, + one_to_many_relationship ) __all__ = ( @@ -81,23 +85,23 @@ def uuid_generator(): return str(uuid4()) -class Blueprint(structures.SQLModelBase): +class Blueprint(SQLModelBase): """ Blueprint model representation. """ __tablename__ = 'blueprints' - storage_id = schema.Column(Integer, primary_key=True, autoincrement=True) - id = schema.Column(Text, index=True) + storage_id = Column(Integer, primary_key=True, autoincrement=True) + id = Column(Text, index=True) - created_at = schema.Column(DateTime, nullable=False, index=True) - main_file_name = schema.Column(Text, nullable=False) - plan = schema.Column(structures.MutableDict.as_mutable(structures.Dict), nullable=False) - updated_at = schema.Column(DateTime) - description = schema.Column(Text) + created_at = Column(DateTime, nullable=False, index=True) + main_file_name = Column(Text, nullable=False) + plan = Column(MutableDict.as_mutable(Dict), nullable=False) + updated_at = Column(DateTime) + description = Column(Text) -class Snapshot(structures.SQLModelBase): +class Snapshot(SQLModelBase): """ Snapshot model representation. """ @@ -111,15 +115,15 @@ class Snapshot(structures.SQLModelBase): STATES = [CREATED, FAILED, CREATING, UPLOADED] END_STATES = [CREATED, FAILED, UPLOADED] - storage_id = schema.Column(Integer, primary_key=True, autoincrement=True) - id = schema.Column(Text, index=True) + storage_id = Column(Integer, primary_key=True, autoincrement=True) + id = Column(Text, index=True) - created_at = schema.Column(DateTime, nullable=False, index=True) - status = schema.Column(Enum(*STATES, name='snapshot_status')) - error = schema.Column(Text) + created_at = Column(DateTime, nullable=False, index=True) + status = Column(Enum(*STATES, name='snapshot_status')) + error = Column(Text) -class Deployment(structures.SQLModelBase): +class Deployment(SQLModelBase): """ Deployment model representation. """ @@ -137,23 +141,23 @@ class Deployment(structures.SQLModelBase): _private_fields = ['blueprint_storage_id'] - storage_id = schema.Column(Integer, primary_key=True, autoincrement=True) - id = schema.Column(Text, index=True) - - created_at = schema.Column(DateTime, nullable=False, index=True) - description = schema.Column(Text) - inputs = schema.Column(structures.MutableDict.as_mutable(structures.Dict)) - groups = schema.Column(structures.MutableDict.as_mutable(structures.Dict)) - permalink = schema.Column(Text) - policy_triggers = schema.Column(structures.MutableDict.as_mutable(structures.Dict)) - policy_types = schema.Column(structures.MutableDict.as_mutable(structures.Dict)) - outputs = schema.Column(structures.MutableDict.as_mutable(structures.Dict)) - scaling_groups = schema.Column(structures.MutableDict.as_mutable(structures.Dict)) - updated_at = schema.Column(DateTime) - workflows = schema.Column(structures.MutableDict.as_mutable(structures.Dict)) - - blueprint_storage_id = structures.foreign_key(Blueprint) - blueprint = structures.one_to_many_relationship( + storage_id = Column(Integer, primary_key=True, autoincrement=True) + id = Column(Text, index=True) + + created_at = Column(DateTime, nullable=False, index=True) + description = Column(Text) + inputs = Column(MutableDict.as_mutable(Dict)) + groups = Column(MutableDict.as_mutable(Dict)) + permalink = Column(Text) + policy_triggers = Column(MutableDict.as_mutable(Dict)) + policy_types = Column(MutableDict.as_mutable(Dict)) + outputs = Column(MutableDict.as_mutable(Dict)) + scaling_groups = Column(MutableDict.as_mutable(Dict)) + updated_at = Column(DateTime) + workflows = Column(MutableDict.as_mutable(Dict)) + + blueprint_storage_id = foreign_key(Blueprint) + blueprint = one_to_many_relationship( child_class_name='Deployment', column_name='blueprint_storage_id', parent_class_name='Blueprint', @@ -169,7 +173,7 @@ class Deployment(structures.SQLModelBase): return self.blueprint.id -class Execution(structures.SQLModelBase): +class Execution(SQLModelBase): """ Execution model representation. """ @@ -203,20 +207,20 @@ class Execution(structures.SQLModelBase): _private_fields = ['deployment_storage_id'] - storage_id = schema.Column(Integer, primary_key=True, autoincrement=True) - id = schema.Column(Text, index=True) + storage_id = Column(Integer, primary_key=True, autoincrement=True) + id = Column(Text, index=True) - created_at = schema.Column(DateTime, index=True) - started_at = schema.Column(DateTime, nullable=True, index=True) - ended_at = schema.Column(DateTime, nullable=True, index=True) - error = schema.Column(Text, nullable=True) - is_system_workflow = schema.Column(Boolean, nullable=False, default=False) - parameters = schema.Column(structures.MutableDict.as_mutable(structures.Dict)) - status = schema.Column(Enum(*STATES, name='execution_status')) - workflow_id = schema.Column(Text, nullable=False) + created_at = Column(DateTime, index=True) + started_at = Column(DateTime, nullable=True, index=True) + ended_at = Column(DateTime, nullable=True, index=True) + error = Column(Text, nullable=True) + is_system_workflow = Column(Boolean, nullable=False, default=False) + parameters = Column(MutableDict.as_mutable(Dict)) + status = Column(Enum(*STATES, name='execution_status')) + workflow_id = Column(Text, nullable=False) - deployment_storage_id = structures.foreign_key(Deployment, nullable=True) - deployment = structures.one_to_many_relationship( + deployment_storage_id = foreign_key(Deployment, nullable=True) + deployment = one_to_many_relationship( child_class_name='Execution', column_name='deployment_storage_id', parent_class_name='Deployment', @@ -249,7 +253,7 @@ class Execution(structures.SQLModelBase): ) -class DeploymentUpdate(structures.SQLModelBase): +class DeploymentUpdate(SQLModelBase): """ Deployment update model representation. """ @@ -270,28 +274,28 @@ class DeploymentUpdate(structures.SQLModelBase): _private_fields = ['execution_storage_id'] - storage_id = schema.Column(Integer, primary_key=True, autoincrement=True) - id = schema.Column(Text, index=True) + storage_id = Column(Integer, primary_key=True, autoincrement=True) + id = Column(Text, index=True) - created_at = schema.Column(DateTime, nullable=False, index=True) - deployment_plan = schema.Column(structures.MutableDict.as_mutable(structures.Dict)) - deployment_update_node_instances = schema.Column(structures.MutableDict.as_mutable( - structures.Dict)) - deployment_update_deployment = schema.Column(structures.MutableDict.as_mutable(structures.Dict)) - deployment_update_nodes = schema.Column(structures.MutableDict.as_mutable(structures.Dict)) - modified_entity_ids = schema.Column(structures.MutableDict.as_mutable(structures.Dict)) - state = schema.Column(Text) + created_at = Column(DateTime, nullable=False, index=True) + deployment_plan = Column(MutableDict.as_mutable(Dict)) + deployment_update_node_instances = Column(MutableDict.as_mutable( + Dict)) + deployment_update_deployment = Column(MutableDict.as_mutable(Dict)) + deployment_update_nodes = Column(MutableDict.as_mutable(Dict)) + modified_entity_ids = Column(MutableDict.as_mutable(Dict)) + state = Column(Text) - execution_storage_id = structures.foreign_key(Execution, nullable=True) - execution = structures.one_to_many_relationship( + execution_storage_id = foreign_key(Execution, nullable=True) + execution = one_to_many_relationship( child_class_name='DeploymentUpdate', column_name='execution_storage_id', parent_class_name='Execution', back_reference_name='deployment_updates' ) - deployment_storage_id = structures.foreign_key(Deployment) - deployment = structures.one_to_many_relationship( + deployment_storage_id = foreign_key(Deployment) + deployment = one_to_many_relationship( child_class_name='DeploymentUpdate', column_name='deployment_storage_id', parent_class_name='Deployment', @@ -321,7 +325,7 @@ class DeploymentUpdate(structures.SQLModelBase): return dep_update_dict -class DeploymentUpdateStep(structures.SQLModelBase): +class DeploymentUpdateStep(SQLModelBase): """ Deployment update step model representation. """ @@ -338,14 +342,14 @@ class DeploymentUpdateStep(structures.SQLModelBase): _private_fields = ['deployment_update_storage_id'] - id = schema.Column(Integer, primary_key=True, autoincrement=True) + id = Column(Integer, primary_key=True, autoincrement=True) - action = schema.Column(Enum(*ACTION_TYPES, name='action_type')) - entity_id = schema.Column(Text, nullable=False) - entity_type = schema.Column(Enum(*ENTITY_TYPES, name='entity_type')) + action = Column(Enum(*ACTION_TYPES, name='action_type')) + entity_id = Column(Text, nullable=False) + entity_type = Column(Enum(*ENTITY_TYPES, name='entity_type')) - deployment_update_storage_id = structures.foreign_key(DeploymentUpdate) - deployment_update = structures.one_to_many_relationship( + deployment_update_storage_id = foreign_key(DeploymentUpdate) + deployment_update = one_to_many_relationship( child_class_name='DeploymentUpdateStep', column_name='deployment_update_storage_id', parent_class_name='DeploymentUpdate', @@ -361,7 +365,7 @@ class DeploymentUpdateStep(structures.SQLModelBase): return self.deployment_update.id -class DeploymentModification(structures.SQLModelBase): +class DeploymentModification(SQLModelBase): """ Deployment modification model representation. """ @@ -385,19 +389,19 @@ class DeploymentModification(structures.SQLModelBase): _private_fields = ['deployment_storage_id'] - storage_id = schema.Column(Integer, primary_key=True, autoincrement=True) - id = schema.Column(Text, index=True) + storage_id = Column(Integer, primary_key=True, autoincrement=True) + id = Column(Text, index=True) - context = schema.Column(structures.MutableDict.as_mutable(structures.Dict)) - created_at = schema.Column(DateTime, nullable=False, index=True) - ended_at = schema.Column(DateTime, index=True) - modified_nodes = schema.Column(structures.MutableDict.as_mutable(structures.Dict)) - node_instances = schema.Column(structures.MutableDict.as_mutable(structures.Dict)) - status = schema.Column( + context = Column(MutableDict.as_mutable(Dict)) + created_at = Column(DateTime, nullable=False, index=True) + ended_at = Column(DateTime, index=True) + modified_nodes = Column(MutableDict.as_mutable(Dict)) + node_instances = Column(MutableDict.as_mutable(Dict)) + status = Column( Enum(*STATES, name='deployment_modification_status')) - deployment_storage_id = structures.foreign_key(Deployment) - deployment = structures.one_to_many_relationship( + deployment_storage_id = foreign_key(Deployment) + deployment = one_to_many_relationship( child_class_name='DeploymentModification', column_name='deployment_storage_id', parent_class_name='Deployment', @@ -413,7 +417,7 @@ class DeploymentModification(structures.SQLModelBase): return self.deployment.id -class Node(structures.SQLModelBase): +class Node(SQLModelBase): """ Node model representation. """ @@ -435,26 +439,26 @@ class Node(structures.SQLModelBase): _private_fields = ['deployment_storage_id'] - storage_id = schema.Column(Integer, primary_key=True, autoincrement=True) - id = schema.Column(Text, index=True) + storage_id = Column(Integer, primary_key=True, autoincrement=True) + id = Column(Text, index=True) - deploy_number_of_instances = schema.Column(Integer, nullable=False) + deploy_number_of_instances = Column(Integer, nullable=False) # TODO: This probably should be a foreign key, but there's no guarantee # in the code, currently, that the host will be created beforehand - host_id = schema.Column(Text) - max_number_of_instances = schema.Column(Integer, nullable=False) - min_number_of_instances = schema.Column(Integer, nullable=False) - number_of_instances = schema.Column(Integer, nullable=False) - planned_number_of_instances = schema.Column(Integer, nullable=False) - plugins = schema.Column(structures.MutableDict.as_mutable(structures.Dict)) - plugins_to_install = schema.Column(structures.MutableDict.as_mutable(structures.Dict)) - properties = schema.Column(structures.MutableDict.as_mutable(structures.Dict)) - operations = schema.Column(structures.MutableDict.as_mutable(structures.Dict)) - type = schema.Column(Text, nullable=False, index=True) - type_hierarchy = schema.Column(PickleType) - - deployment_storage_id = structures.foreign_key(Deployment) - deployment = structures.one_to_many_relationship( + host_id = Column(Text) + max_number_of_instances = Column(Integer, nullable=False) + min_number_of_instances = Column(Integer, nullable=False) + number_of_instances = Column(Integer, nullable=False) + planned_number_of_instances = Column(Integer, nullable=False) + plugins = Column(MutableDict.as_mutable(Dict)) + plugins_to_install = Column(MutableDict.as_mutable(Dict)) + properties = Column(MutableDict.as_mutable(Dict)) + operations = Column(MutableDict.as_mutable(Dict)) + type = Column(Text, nullable=False, index=True) + type_hierarchy = Column(PickleType) + + deployment_storage_id = foreign_key(Deployment) + deployment = one_to_many_relationship( child_class_name='Node', column_name='deployment_storage_id', parent_class_name='Deployment', @@ -478,7 +482,7 @@ class Node(structures.SQLModelBase): return self.deployment.blueprint_id -class Relationship(structures.SQLModelBase): +class Relationship(SQLModelBase): """ Relationship model representation. """ @@ -498,27 +502,27 @@ class Relationship(structures.SQLModelBase): _private_fields = ['relationship_storage_source_node_id', 'relationship_storage_target_node_id'] - storage_id = schema.Column(Integer, primary_key=True, autoincrement=True) - id = schema.Column(Text, index=True) + storage_id = Column(Integer, primary_key=True, autoincrement=True) + id = Column(Text, index=True) - source_interfaces = schema.Column(structures.MutableDict.as_mutable(structures.Dict)) - source_operations = schema.Column(structures.MutableDict.as_mutable(structures.Dict)) - target_interfaces = schema.Column(structures.MutableDict.as_mutable(structures.Dict)) - target_operations = schema.Column(structures.MutableDict.as_mutable(structures.Dict)) - type = schema.Column(String) - type_hierarchy = schema.Column(PickleType) # TODO: this should be list - properties = schema.Column(structures.MutableDict.as_mutable(structures.Dict)) + source_interfaces = Column(MutableDict.as_mutable(Dict)) + source_operations = Column(MutableDict.as_mutable(Dict)) + target_interfaces = Column(MutableDict.as_mutable(Dict)) + target_operations = Column(MutableDict.as_mutable(Dict)) + type = Column(String) + type_hierarchy = Column(PickleType) # TODO: this should be list + properties = Column(MutableDict.as_mutable(Dict)) - source_node_storage_id = structures.foreign_key(Node) - target_node_storage_id = structures.foreign_key(Node) + source_node_storage_id = foreign_key(Node) + target_node_storage_id = foreign_key(Node) - source_node = structures.one_to_many_relationship( + source_node = one_to_many_relationship( child_class_name='Relationship', column_name='source_node_storage_id', parent_class_name='Node', back_reference_name='relationship_source' ) - target_node = structures.one_to_many_relationship( + target_node = one_to_many_relationship( child_class_name='Relationship', column_name='target_node_storage_id', parent_class_name='Node', @@ -526,7 +530,7 @@ class Relationship(structures.SQLModelBase): ) -class NodeInstance(structures.SQLModelBase): +class NodeInstance(SQLModelBase): """ Node instance model representation. """ @@ -547,19 +551,19 @@ class NodeInstance(structures.SQLModelBase): _private_fields = ['node_storage_id', 'deployment_storage_id'] - storage_id = schema.Column(Integer, primary_key=True, autoincrement=True) - id = schema.Column(Text, index=True) + storage_id = Column(Integer, primary_key=True, autoincrement=True) + id = Column(Text, index=True) # TODO: This probably should be a foreign key, but there's no guarantee # in the code, currently, that the host will be created beforehand - host_id = schema.Column(Text) - runtime_properties = schema.Column(structures.MutableDict.as_mutable(structures.Dict)) - scaling_groups = schema.Column(structures.MutableDict.as_mutable(structures.Dict)) - state = schema.Column(Text, nullable=False) - version = schema.Column(Integer, default=1) - - node_storage_id = structures.foreign_key(Node) - node = structures.one_to_many_relationship( + host_id = Column(Text) + runtime_properties = Column(MutableDict.as_mutable(Dict)) + scaling_groups = Column(MutableDict.as_mutable(Dict)) + state = Column(Text, nullable=False) + version = Column(Integer, default=1) + + node_storage_id = foreign_key(Node) + node = one_to_many_relationship( child_class_name='NodeInstance', column_name='node_storage_id', parent_class_name='Node', @@ -574,8 +578,8 @@ class NodeInstance(structures.SQLModelBase): """ return self.node.id - deployment_storage_id = structures.foreign_key(Deployment) - deployment = structures.one_to_many_relationship( + deployment_storage_id = foreign_key(Deployment) + deployment = one_to_many_relationship( child_class_name='NodeInstance', column_name='deployment_storage_id', parent_class_name='Deployment', @@ -583,7 +587,7 @@ class NodeInstance(structures.SQLModelBase): ) -class RelationshipInstance(structures.SQLModelBase): +class RelationshipInstance(SQLModelBase): """ Relationship instance model representation. """ @@ -605,27 +609,27 @@ class RelationshipInstance(structures.SQLModelBase): 'source_node_instance_id', 'target_node_instance_id'] - storage_id = schema.Column(Integer, primary_key=True, autoincrement=True) - id = schema.Column(Text, index=True) + storage_id = Column(Integer, primary_key=True, autoincrement=True) + id = Column(Text, index=True) - type = schema.Column(String) + type = Column(String) - source_node_instance_storage_id = structures.foreign_key(NodeInstance) - source_node_instance = structures.one_to_many_relationship( + source_node_instance_storage_id = foreign_key(NodeInstance) + source_node_instance = one_to_many_relationship( child_class_name='RelationshipInstance', column_name='source_node_instance_storage_id', parent_class_name='NodeInstance', back_reference_name='relationship_instance_source' ) - target_node_instance_storage_id = structures.foreign_key(NodeInstance) - target_node_instance = structures.one_to_many_relationship( + target_node_instance_storage_id = foreign_key(NodeInstance) + target_node_instance = one_to_many_relationship( child_class_name='RelationshipInstance', column_name='target_node_instance_storage_id', parent_class_name='NodeInstance', back_reference_name='relationship_instance_target' ) - relationship_storage_id = structures.foreign_key(Relationship) - relationship = structures.one_to_many_relationship( + relationship_storage_id = foreign_key(Relationship) + relationship = one_to_many_relationship( child_class_name='RelationshipInstance', column_name='relationship_storage_id', parent_class_name='Relationship', @@ -633,41 +637,41 @@ class RelationshipInstance(structures.SQLModelBase): ) -class ProviderContext(structures.SQLModelBase): +class ProviderContext(SQLModelBase): """ Provider context model representation. """ __tablename__ = 'provider_context' - id = schema.Column(Text, primary_key=True) - name = schema.Column(Text, nullable=False) - context = schema.Column(structures.MutableDict.as_mutable(structures.Dict), nullable=False) + id = Column(Text, primary_key=True) + name = Column(Text, nullable=False) + context = Column(MutableDict.as_mutable(Dict), nullable=False) -class Plugin(structures.SQLModelBase): +class Plugin(SQLModelBase): """ Plugin model representation. """ __tablename__ = 'plugins' - storage_id = schema.Column(Integer, primary_key=True, autoincrement=True) - id = schema.Column(Text, index=True) + storage_id = Column(Integer, primary_key=True, autoincrement=True) + id = Column(Text, index=True) - archive_name = schema.Column(Text, nullable=False, index=True) - distribution = schema.Column(Text) - distribution_release = schema.Column(Text) - distribution_version = schema.Column(Text) - excluded_wheels = schema.Column(structures.MutableDict.as_mutable(structures.Dict)) - package_name = schema.Column(Text, nullable=False, index=True) - package_source = schema.Column(Text) - package_version = schema.Column(Text) - supported_platform = schema.Column(structures.MutableDict.as_mutable(structures.Dict)) - supported_py_versions = schema.Column(structures.MutableDict.as_mutable(structures.Dict)) - uploaded_at = schema.Column(DateTime, nullable=False, index=True) - wheels = schema.Column(structures.MutableDict.as_mutable(structures.Dict), nullable=False) + archive_name = Column(Text, nullable=False, index=True) + distribution = Column(Text) + distribution_release = Column(Text) + distribution_version = Column(Text) + excluded_wheels = Column(MutableDict.as_mutable(Dict)) + package_name = Column(Text, nullable=False, index=True) + package_source = Column(Text) + package_version = Column(Text) + supported_platform = Column(MutableDict.as_mutable(Dict)) + supported_py_versions = Column(MutableDict.as_mutable(Dict)) + uploaded_at = Column(DateTime, nullable=False, index=True) + wheels = Column(MutableDict.as_mutable(Dict), nullable=False) -class Task(structures.SQLModelBase): +class Task(SQLModelBase): """ A Model which represents an task """ @@ -705,35 +709,35 @@ class Task(structures.SQLModelBase): INFINITE_RETRIES = -1 - id = schema.Column(String, primary_key=True, default=uuid_generator) - status = schema.Column(Enum(*STATES), name='status', default=PENDING) + id = Column(String, primary_key=True, default=uuid_generator) + status = Column(Enum(*STATES), name='status', default=PENDING) - execution_id = schema.Column(String) - due_at = schema.Column(DateTime, default=datetime.utcnow, nullable=True) - started_at = schema.Column(DateTime, default=None, nullable=True) - ended_at = schema.Column(DateTime, default=None, nullable=True) + execution_id = Column(String) + due_at = Column(DateTime, default=datetime.utcnow, nullable=True) + started_at = Column(DateTime, default=None, nullable=True) + ended_at = Column(DateTime, default=None, nullable=True) # , validation_func=_Validation.validate_max_attempts) - max_attempts = schema.Column(Integer, default=1) - retry_count = schema.Column(Integer, default=0) - retry_interval = schema.Column(Float, default=0) - ignore_failure = schema.Column(Boolean, default=False) + max_attempts = Column(Integer, default=1) + retry_count = Column(Integer, default=0) + retry_interval = Column(Float, default=0) + ignore_failure = Column(Boolean, default=False) # Operation specific fields - name = schema.Column(String) - operation_mapping = schema.Column(String) - inputs = schema.Column(structures.MutableDict.as_mutable(structures.Dict)) + name = Column(String) + operation_mapping = Column(String) + inputs = Column(MutableDict.as_mutable(Dict)) - node_instance_storage_id = structures.foreign_key(NodeInstance, nullable=True) - relationship_instance_storage_id = structures.foreign_key(RelationshipInstance, nullable=True) + node_instance_storage_id = foreign_key(NodeInstance, nullable=True) + relationship_instance_storage_id = foreign_key(RelationshipInstance, nullable=True) - node_instance = structures.one_to_many_relationship( + node_instance = one_to_many_relationship( child_class_name='Task', column_name='node_instance_storage_id', parent_class_name='NodeInstance', back_reference_name='tasks', ) - relationship_instance = structures.one_to_many_relationship( + relationship_instance = one_to_many_relationship( child_class_name='Task', column_name='relationship_instance_storage_id', parent_class_name='RelationshipInstance', http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/749f3755/aria/storage/structures.py ---------------------------------------------------------------------- diff --git a/aria/storage/structures.py b/aria/storage/structures.py index 92434d6..4d86bf8 100644 --- a/aria/storage/structures.py +++ b/aria/storage/structures.py @@ -29,11 +29,13 @@ classes: import json import jsonpickle -from flask_sqlalchemy import SQLAlchemy +from sqlalchemy import VARCHAR from sqlalchemy.ext.mutable import Mutable +from sqlalchemy.ext.declarative import declarative_base # pylint: disable=unused-import from sqlalchemy import ( schema, + Column, Integer, Text, DateTime, @@ -43,13 +45,12 @@ from sqlalchemy import ( PickleType, Float, TypeDecorator, - ForeignKey + ForeignKey, + orm, ) - -db = SQLAlchemy() - +Model = declarative_base() class classproperty(object): """A class that acts a a decorator for class-level properties @@ -87,7 +88,7 @@ def foreign_key( :param column_type: The type (integer/text/etc.) of the column :return: """ - return db.Column( + return Column( column_type, ForeignKey( '{0}.{1}'.format(parent_table.__tablename__, id_col_name), @@ -114,7 +115,7 @@ def one_to_many_relationship( :param parent_id_name: Name of the parent table's ID column [default: `id`] :return: """ - return db.relationship( + return orm.relationship( parent_class_name, primaryjoin='{0}.{1} == {2}.{3}'.format( child_class_name, @@ -124,7 +125,7 @@ def one_to_many_relationship( ), # The following line make sure that when the *parent* is # deleted, all its connected children are deleted as well - backref=db.backref(back_reference_name, cascade='all') + backref=orm.backref(back_reference_name, cascade='all') ) @@ -141,42 +142,19 @@ def many_to_many_relationship( current table from the other table :return: """ - return db.relationship( + return orm.relationship( other_table_class_name, secondary=connecting_table, - backref=db.backref(back_reference_name, lazy='dynamic') + backref=orm.backref(back_reference_name, lazy='dynamic') ) -# class UTCDateTime(TypeDecorator): -# -# impl = db.DateTime -# -# def process_result_value(self, value, engine): -# # Adhering to the same norms used in the rest of the code -# if value is not None: -# # When the date has a microsecond value equal to 0, -# # isoformat returns the time as 17:22:11 instead of -# # 17:22:11.000, so we need to adjust the returned value -# if value.microsecond: -# return '{0}Z'.format(value.isoformat()[:-3]) -# else: -# return '{0}.000Z'.format(value.isoformat()) -# -# def process_bind_param(self, value, dialect): -# if isinstance(value, basestring): -# # SQLite only accepts datetime objects -# return date_parser.parse(value) -# else: -# return value - - class Dict(TypeDecorator): """ Dict represenation of type. """ - impl = db.VARCHAR + impl = VARCHAR def process_bind_param(self, value, dialect): if value is not None: @@ -219,7 +197,7 @@ class MutableDict(Mutable, dict): self.changed() -class SQLModelBase(db.Model): +class SQLModelBase(Model): """Abstract base class for all SQL models that allows [de]serialization """ # SQLAlchemy syntax @@ -274,3 +252,6 @@ class SQLModelBase(db.Model): def __unicode__(self): return str(self) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.to_dict == other.to_dict http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/749f3755/tests/mock/context.py ---------------------------------------------------------------------- diff --git a/tests/mock/context.py b/tests/mock/context.py index 54e877c..7f09b75 100644 --- a/tests/mock/context.py +++ b/tests/mock/context.py @@ -14,20 +14,25 @@ # limitations under the License. import pytest -from flask import Flask +from sqlalchemy import create_engine, orm from aria import application_model_storage from aria.orchestrator import context from aria.storage.mapi import SQLAlchemyModelAPI -from tests import storage from . import models @pytest.fixture def simple(**kwargs): - storage.drop_tables() - model_storage = application_model_storage(SQLAlchemyModelAPI, api_params=dict(app=Flask('app'))) + engine = create_engine('sqlite:///:memory:') + model_storage = application_model_storage( + SQLAlchemyModelAPI, + api_params={ + 'engine': engine, + 'session': orm.sessionmaker(bind=engine)() + } + ) model_storage.blueprint.store(models.get_blueprint()) blueprint = model_storage.blueprint.get(models.BLUEPRINT_ID) deployment = models.get_deployment(blueprint) http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/749f3755/tests/orchestrator/context/test_operation.py ---------------------------------------------------------------------- diff --git a/tests/orchestrator/context/test_operation.py b/tests/orchestrator/context/test_operation.py index 4f52577..ec13154 100644 --- a/tests/orchestrator/context/test_operation.py +++ b/tests/orchestrator/context/test_operation.py @@ -35,7 +35,9 @@ global_test_holder = {} @pytest.fixture def ctx(): - return mock.context.simple() + context = mock.context.simple() + yield context + context.model.drop() @pytest.fixture http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/749f3755/tests/orchestrator/context/test_toolbelt.py ---------------------------------------------------------------------- diff --git a/tests/orchestrator/context/test_toolbelt.py b/tests/orchestrator/context/test_toolbelt.py index dc06c4e..d5297ce 100644 --- a/tests/orchestrator/context/test_toolbelt.py +++ b/tests/orchestrator/context/test_toolbelt.py @@ -33,8 +33,9 @@ global_test_holder = {} @pytest.fixture def workflow_context(): - return mock.context.simple() - + context = mock.context.simple() + yield context + context.model.drop() @pytest.fixture def executor(): http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/749f3755/tests/orchestrator/context/test_workflow.py ---------------------------------------------------------------------- diff --git a/tests/orchestrator/context/test_workflow.py b/tests/orchestrator/context/test_workflow.py index f385b93..03648f1 100644 --- a/tests/orchestrator/context/test_workflow.py +++ b/tests/orchestrator/context/test_workflow.py @@ -59,8 +59,7 @@ class TestWorkflowContext(object): @pytest.fixture(scope='function') def storage(): - test_storage.drop_tables() - result = application_model_storage(SQLAlchemyModelAPI, api_params=dict(app=Flask('app'))) + result = application_model_storage(SQLAlchemyModelAPI) result.blueprint.store(models.get_blueprint()) blueprint = result.blueprint.get(models.BLUEPRINT_ID) result.deployment.store(models.get_deployment(blueprint)) http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/749f3755/tests/storage/__init__.py ---------------------------------------------------------------------- diff --git a/tests/storage/__init__.py b/tests/storage/__init__.py index 5cbdc6a..f5939da 100644 --- a/tests/storage/__init__.py +++ b/tests/storage/__init__.py @@ -16,8 +16,6 @@ from shutil import rmtree from tempfile import mkdtemp -from aria import storage - class TestFileSystem(object): @@ -26,8 +24,3 @@ class TestFileSystem(object): def teardown_method(self): rmtree(self.path) - - -def drop_tables(): - if storage.structures.db.app is not None: - storage.structures.db.drop_all(bind=None) http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/749f3755/tests/storage/test_model_storage.py ---------------------------------------------------------------------- diff --git a/tests/storage/test_model_storage.py b/tests/storage/test_model_storage.py index 9723afc..6e36f84 100644 --- a/tests/storage/test_model_storage.py +++ b/tests/storage/test_model_storage.py @@ -17,7 +17,6 @@ import tempfile import shutil import pytest -from flask import Flask from aria import application_model_storage @@ -34,8 +33,8 @@ from . import drop_tables temp_dir = tempfile.mkdtemp() APIs = [ - # (storage_api.FileSystemModelAPI, dict(directory=temp_dir)), - (storage_api.SQLAlchemyModelAPI, dict(app=Flask('app'))) + ModelStorage(storage_api.SQLAlchemyModelAPI), + # ModelStorage(storage_api.FileSystemModelAPI, api_params=dict(directory=temp_dir)), ] @@ -49,19 +48,14 @@ def cleanup(): pass -@pytest.mark.parametrize('api, api_params', APIs) -def test_storage_base(api, api_params): - storage = ModelStorage(api, api_params=api_params) - - assert storage.api == api - +@pytest.mark.parametrize('storage', APIs) +def test_storage_base(storage): with pytest.raises(AttributeError): storage.non_existent_attribute() -@pytest.mark.parametrize('api, api_params', APIs) -def test_model_storage(api, api_params): - storage = ModelStorage(api, api_params=api_params) +@pytest.mark.parametrize('storage', APIs) +def test_model_storage(storage): storage.register(models.ProviderContext) pc = models.ProviderContext(context={}, name='context_name', id='id1') @@ -82,9 +76,8 @@ def test_model_storage(api, api_params): storage.provider_context.get('id1') -@pytest.mark.parametrize('api, api_params', APIs) -def test_storage_driver(api, api_params): - storage = ModelStorage(api, api_params=api_params) +@pytest.mark.parametrize('storage', APIs) +def test_storage_driver(storage): storage.register(models.ProviderContext) pc = models.ProviderContext(context={}, name='context_name', id='id2') @@ -101,17 +94,17 @@ def test_storage_driver(api, api_params): storage.registered['provider_context'].get('id2') -@pytest.mark.parametrize('api, api_params', APIs) -def test_application_storage_factory(api, api_params): - storage = application_model_storage(api, api_params=api_params) - assert storage.node - assert storage.node_instance - assert storage.plugin - assert storage.blueprint - assert storage.snapshot - assert storage.deployment - assert storage.deployment_update - assert storage.deployment_update_step - assert storage.deployment_modification - assert storage.execution - assert storage.provider_context +# @pytest.mark.parametrize('storage', APIs) +# def test_application_storage_factory(storage): +# storage = application_model_storage(api, api_params=api_params) +# assert storage.node +# assert storage.node_instance +# assert storage.plugin +# assert storage.blueprint +# assert storage.snapshot +# assert storage.deployment +# assert storage.deployment_update +# assert storage.deployment_update_step +# assert storage.deployment_modification +# assert storage.execution +# assert storage.provider_context
