Repository: incubator-ariatosca Updated Branches: refs/heads/ARIA-39-Genericize-storage-models 17e02e5e2 -> ee4209616 (forced update)
cleaned up relationship and foreign key creation Project: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/commit/ee420961 Tree: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/tree/ee420961 Diff: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/diff/ee420961 Branch: refs/heads/ARIA-39-Genericize-storage-models Commit: ee4209616c682d5f1cc9bc219b4d39cff1748240 Parents: 24b55da Author: mxmrlv <[email protected]> Authored: Mon Dec 19 19:17:40 2016 +0200 Committer: mxmrlv <[email protected]> Committed: Tue Dec 20 11:54:31 2016 +0200 ---------------------------------------------------------------------- aria/orchestrator/context/workflow.py | 7 +- aria/orchestrator/workflows/builtin/heal.py | 9 +- aria/storage/base_model.py | 190 ++++++++++++----------- aria/storage/core.py | 2 +- aria/storage/model.py | 6 +- aria/storage/structure.py | 84 +++++----- tests/orchestrator/context/test_toolbelt.py | 2 +- tests/storage/test_models.py | 15 +- 8 files changed, 166 insertions(+), 149 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/ee420961/aria/orchestrator/context/workflow.py ---------------------------------------------------------------------- diff --git a/aria/orchestrator/context/workflow.py b/aria/orchestrator/context/workflow.py index 1365888..feca33a 100644 --- a/aria/orchestrator/context/workflow.py +++ b/aria/orchestrator/context/workflow.py @@ -74,9 +74,11 @@ class WorkflowContext(BaseContext): """ Iterator over nodes """ + key = 'deployment_{0}'.format(self.model.node_instance.model_cls.user_id_column()) + return self.model.node.iter( filters={ - 'deployment_id': getattr(self.deployment, self.deployment.user_id_column()) + key: getattr(self.deployment, self.deployment.user_id_column()) } ) @@ -85,9 +87,10 @@ class WorkflowContext(BaseContext): """ Iterator over node instances """ + key = 'deployment_{0}'.format(self.model.node_instance.model_cls.user_id_column()) return self.model.node_instance.iter( filters={ - 'deployment_id': getattr(self.deployment, self.deployment.user_id_column()) + key: getattr(self.deployment, self.deployment.user_id_column()) } ) http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/ee420961/aria/orchestrator/workflows/builtin/heal.py ---------------------------------------------------------------------- diff --git a/aria/orchestrator/workflows/builtin/heal.py b/aria/orchestrator/workflows/builtin/heal.py index dcf398c..406a42a 100644 --- a/aria/orchestrator/workflows/builtin/heal.py +++ b/aria/orchestrator/workflows/builtin/heal.py @@ -163,10 +163,11 @@ def heal_install(ctx, graph, failing_node_instances, targeted_node_instances): def _get_contained_subgraph(context, host_node_instance): - contained_instances = [node_instance - for node_instance in context.node_instances - if node_instance.host_id == host_node_instance.id and - node_instance.id != node_instance.host_id] + contained_instances = [ + node_instance + for node_instance in context.node_instances + if node_instance == host_node_instance and node_instance != node_instance + ] result = [host_node_instance] if not contained_instances: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/ee420961/aria/storage/base_model.py ---------------------------------------------------------------------- diff --git a/aria/storage/base_model.py b/aria/storage/base_model.py index 1f1a80c..352f121 100644 --- a/aria/storage/base_model.py +++ b/aria/storage/base_model.py @@ -112,14 +112,14 @@ class DeploymentBase(ModelBase): @declared_attr def blueprint_fk(cls): - return cls.foreign_key('blueprints', nullable=False) + return cls.foreign_key(BlueprintBase, nullable=False) @declared_attr def blueprint(cls): - return cls.one_to_many_relationship('blueprint_fk', 'Blueprint') + return cls.one_to_many_relationship('blueprint_fk') @declared_attr - def blueprint_id(cls): + def blueprint_name(cls): return association_proxy('blueprint', cls.user_id_column()) @@ -175,21 +175,26 @@ class ExecutionBase(ModelBase): status = Column(Enum(*STATES, name='execution_status'), default=PENDING) workflow_name = Column(Text) - blueprint = association_proxy('deployment', 'blueprint') - blueprint_id = association_proxy('deployment', 'blueprint_id') + @declared_attr + def blueprint(cls): + return association_proxy('deployment', 'blueprint') @declared_attr def deployment_fk(cls): - return cls.foreign_key('deployments', nullable=True) + return cls.foreign_key(DeploymentBase, nullable=True) @declared_attr def deployment(cls): - return cls.one_to_many_relationship('deployment_fk', 'Deployment') + return cls.one_to_many_relationship('deployment_fk') @declared_attr - def deployment_id(cls): + def deployment_name(cls): return association_proxy('deployment', cls.user_id_column()) + @declared_attr + def blueprint_name(cls): + return association_proxy('deployment', 'blueprint_name') + def __str__(self): return '<{0} id=`{1}` (status={2})>'.format( self.__class__.__name__, @@ -219,27 +224,27 @@ class DeploymentUpdateBase(ModelBase): @declared_attr def execution_fk(cls): - return cls.foreign_key('executions', nullable=True) + return cls.foreign_key(ExecutionBase, nullable=True) @declared_attr - def execution_id(cls): - return association_proxy('execution', cls.user_id_column()) + def execution(cls): + return cls.one_to_many_relationship('execution_fk') @declared_attr - def execution(cls): - return cls.one_to_many_relationship('execution_fk', 'Execution') + def execution_name(cls): + return association_proxy('execution', cls.user_id_column()) @declared_attr def deployment_fk(cls): - return cls.foreign_key('deployments') + return cls.foreign_key(DeploymentBase) @declared_attr - def deployment_id(cls): - return association_proxy('deployment', cls.user_id_column()) + def deployment(cls): + return cls.one_to_many_relationship('deployment_fk') @declared_attr - def deployment(cls): - return cls.one_to_many_relationship('deployment_fk', 'Deployment') + def deployment_name(cls): + return association_proxy('deployment', cls.user_id_column()) def to_dict(self, suppress_error=False, **kwargs): dep_update_dict = super(DeploymentUpdateBase, self).to_dict(suppress_error) #pylint: disable=no-member @@ -277,25 +282,23 @@ class DeploymentUpdateStepBase(ModelBase): PLUGIN='plugin' ) - action = Column(Enum(*ACTION_TYPES, name='action_type'), nullable=False) entity_id = Column(Text, nullable=False) entity_type = Column(Enum(*ENTITY_TYPES, name='entity_type'), nullable=False) @declared_attr def deployment_update_fk(cls): - return cls.foreign_key('deployment_updates') - - @declared_attr - def deployment_update_id(cls): - return association_proxy('deployment_update', cls.user_id_column()) + return cls.foreign_key(DeploymentUpdateBase) @declared_attr def deployment_update(cls): return cls.one_to_many_relationship('deployment_update_fk', - 'DeploymentUpdate', backreference='steps') + @declared_attr + def deployment_update_name(cls): + return association_proxy('deployment_update', cls.user_id_column()) + def __hash__(self): return hash((self.id, self.entity_id)) @@ -346,19 +349,18 @@ class DeploymentModificationBase(ModelBase): status = Column(Enum(*STATES, name='deployment_modification_status')) @declared_attr - def deployment_id(cls): - return association_proxy('deployment', cls.user_id_column()) - - @declared_attr def deployment_fk(cls): - return cls.foreign_key('deployments') + return cls.foreign_key(DeploymentBase) @declared_attr def deployment(cls): return cls.one_to_many_relationship('deployment_fk', - 'Deployment', backreference='modifications') + @declared_attr + def deployment_name(cls): + return association_proxy('deployment', cls.user_id_column()) + class NodeBase(ModelBase): """ @@ -372,30 +374,32 @@ class NodeBase(ModelBase): _private_fields = ['blueprint_fk', 'host_fk'] @declared_attr - def host_id(cls): - return association_proxy('host', cls.user_id_column()) - - @declared_attr def host_fk(cls): - return cls.foreign_key('nodes', nullable=True) + return cls.foreign_key(NodeBase, nullable=True) @declared_attr def host(cls): return cls.relationship_to_self('host_fk') @declared_attr - def deployment_id(cls): - return association_proxy('deployment', cls.user_id_column()) + def host_name(cls): + return association_proxy('host', cls.user_id_column()) @declared_attr def deployment_fk(cls): - return cls.foreign_key('deployments') + return cls.foreign_key(DeploymentBase) @declared_attr def deployment(cls): - return cls.one_to_many_relationship('deployment_fk', 'Deployment') + return cls.one_to_many_relationship('deployment_fk') + + @declared_attr + def deployment_name(cls): + return association_proxy('deployment', cls.user_id_column()) - blueprint_id = association_proxy('deployment', 'blueprint_id') + @declared_attr + def blueprint_name(cls): + return association_proxy('deployment', 'blueprint_{0}'.format(cls.user_id_column())) deploy_number_of_instances = Column(Integer, nullable=False) max_number_of_instances = Column(Integer, nullable=False) @@ -416,36 +420,34 @@ class RelationshipBase(ModelBase): """ __tablename__ = 'relationships' - _private_fields = ['source_node_id', 'target_node_id'] - - @declared_attr - def source_id(cls): - return association_proxy('source_node', cls.user_id_column()) + _private_fields = ['source_node_fk', 'target_node_fk'] @declared_attr def source_node_fk(cls): - return cls.foreign_key('nodes') + return cls.foreign_key(NodeBase) @declared_attr def source_node(cls): return cls.one_to_many_relationship('source_node_fk', - 'Node', backreference='outbound_relationships') @declared_attr - def target_id(cls): - return association_proxy('target_node', cls.user_id_column()) + def source_name(cls): + return association_proxy('source_node', cls.user_id_column()) @declared_attr def target_node_fk(cls): - return cls.foreign_key('nodes') + return cls.foreign_key(NodeBase) @declared_attr def target_node(cls): return cls.one_to_many_relationship('target_node_fk', - 'Node', backreference='inbound_relationships') + @declared_attr + def target_name(cls): + return association_proxy('target_node', cls.user_id_column()) + source_interfaces = Column(Dict) source_operations = Column(Dict, nullable=False) target_interfaces = Column(Dict) @@ -468,32 +470,36 @@ class NodeInstanceBase(ModelBase): version = Column(Integer, default=1) @declared_attr - def host_id(cls): - return association_proxy('host', cls.user_id_column()) - - @declared_attr def host_fk(cls): - return cls.foreign_key('node_instances', nullable=True) + return cls.foreign_key(NodeInstanceBase, nullable=True) @declared_attr def host(cls): return cls.relationship_to_self('host_fk') - deployment = association_proxy('node', 'deployment') - deployment_id = association_proxy('node', 'deployment_id') - deployment_name = association_proxy('node', 'deployment_name') + @declared_attr + def host_name(cls): + return association_proxy('host', cls.user_id_column()) @declared_attr - def node_id(cls): - return association_proxy('node', cls.user_id_column()) + def deployment(cls): + return association_proxy('node', 'deployment') + + @declared_attr + def deployment_name(cls): + return association_proxy('node', 'deployment_name') @declared_attr def node_fk(cls): - return cls.foreign_key('nodes', nullable=True) + return cls.foreign_key(NodeBase, nullable=True) @declared_attr def node(cls): - return cls.one_to_many_relationship('node_fk', 'Node') + return cls.one_to_many_relationship('node_fk') + + @declared_attr + def node_name(cls): + return association_proxy('node', cls.user_id_column()) class RelationshipInstanceBase(ModelBase): @@ -501,49 +507,47 @@ class RelationshipInstanceBase(ModelBase): Relationship instance model representation. """ __tablename__ = 'relationship_instances' - _private_fields = ['relationship_storage_id', - 'source_node_instance_id', - 'target_node_instance_id'] - - @declared_attr - def source_node_instance_id(cls): - return association_proxy('source_node_instance', cls.user_id_column()) + _private_fields = ['relationship_storage_fk', + 'source_node_instance_fk', + 'target_node_instance_fk'] @declared_attr def source_node_instance_fk(cls): - return cls.foreign_key('node_instances') + return cls.foreign_key(NodeInstanceBase) @declared_attr def source_node_instance(cls): return cls.one_to_many_relationship('source_node_instance_fk', - 'NodeInstance', backreference='outbound_relationship_instances') @declared_attr - def target_node_instance_id(cls): - return association_proxy('target_node_instance', cls.user_id_column()) + def source_node_instance_name(cls): + return association_proxy('source_node_instance', cls.user_id_column()) @declared_attr def target_node_instance_fk(cls): - return cls.foreign_key('node_instances') + return cls.foreign_key(NodeInstanceBase) @declared_attr def target_node_instance(cls): return cls.one_to_many_relationship('target_node_instance_fk', - 'NodeInstance', backreference='inbound_relationship_instances') @declared_attr - def relationship_fk(cls): - return cls.foreign_key('relationships') + def target_node_instance_name(cls): + return association_proxy('target_node_instance', cls.user_id_column()) @declared_attr - def relationship_id(cls): - return association_proxy('relationship', cls.user_id_column()) + def relationship_fk(cls): + return cls.foreign_key(RelationshipBase) @declared_attr def relationship(cls): - return cls.one_to_many_relationship('relationship_fk', 'Relationship') + return cls.one_to_many_relationship('relationship_fk') + + @declared_attr + def relationship_name(cls): + return association_proxy('relationship', cls.user_id_column()) class PluginBase(ModelBase): @@ -575,27 +579,27 @@ class TaskBase(ModelBase): @declared_attr def node_instance_fk(cls): - return cls.foreign_key('node_instances', nullable=True) + return cls.foreign_key(NodeInstanceBase, nullable=True) @declared_attr - def node_instance_id(cls): + def node_instance_name(cls): return association_proxy('node_instance', cls.user_id_column()) @declared_attr def node_instance(cls): - return cls.one_to_many_relationship('node_instance_fk', 'NodeInstance') + return cls.one_to_many_relationship('node_instance_fk') @declared_attr def relationship_instance_fk(cls): - return cls.foreign_key('relationship_instances', nullable=True) + return cls.foreign_key(RelationshipInstanceBase, nullable=True) @declared_attr - def relationship_instance_id(cls): + def relationship_instance_name(cls): return association_proxy('relationship_instance', cls.user_id_column()) @declared_attr def relationship_instance(cls): - return cls.one_to_many_relationship('relationship_instance_fk', 'RelationshipInstance') + return cls.one_to_many_relationship('relationship_instance_fk') PENDING = 'pending' RETRYING = 'retrying' @@ -641,15 +645,15 @@ class TaskBase(ModelBase): @declared_attr def execution_fk(cls): - return cls.foreign_key('executions', nullable=True) + return cls.foreign_key(ExecutionBase, nullable=True) @declared_attr - def execution_id(cls): - return association_proxy('execution', cls.user_id_column()) + def execution(cls): + return cls.one_to_many_relationship('execution_fk') @declared_attr - def execution(cls): - return cls.one_to_many_relationship('execution_fk', 'Execution') + def execution_name(cls): + return association_proxy('execution', cls.user_id_column()) @property def actor(self): http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/ee420961/aria/storage/core.py ---------------------------------------------------------------------- diff --git a/aria/storage/core.py b/aria/storage/core.py index a5d3210..94b4fe0 100644 --- a/aria/storage/core.py +++ b/aria/storage/core.py @@ -53,9 +53,9 @@ class Storage(LoggerMixin): Represents the storage """ def __init__(self, api_cls, api_kwargs=None, items=(), **kwargs): - self._api_kwargs = api_kwargs or {} super(Storage, self).__init__(**kwargs) self.api = api_cls + self._api_kwargs = api_kwargs or {} self.registered = {} for item in items: self.register(item) http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/ee420961/aria/storage/model.py ---------------------------------------------------------------------- diff --git a/aria/storage/model.py b/aria/storage/model.py index 52f2612..61795aa 100644 --- a/aria/storage/model.py +++ b/aria/storage/model.py @@ -36,8 +36,9 @@ classes: * ProviderContext - provider context implementation model. * Plugin - plugin implementation model. """ - +from sqlalchemy.ext.associationproxy import association_proxy from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy.ext.declarative import declared_attr from . import structure from . import base_model as base @@ -67,7 +68,6 @@ class Blueprint(DeclarativeBase, base.BlueprintBase): class Deployment(DeclarativeBase, base.DeploymentBase): pass - class Execution(DeclarativeBase, base.ExecutionBase): pass @@ -105,4 +105,4 @@ class Plugin(DeclarativeBase, base.PluginBase): class Task(DeclarativeBase, base.TaskBase): - pass + pass \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/ee420961/aria/storage/structure.py ---------------------------------------------------------------------- diff --git a/aria/storage/structure.py b/aria/storage/structure.py index 2f6186d..0e99e2b 100644 --- a/aria/storage/structure.py +++ b/aria/storage/structure.py @@ -28,6 +28,7 @@ classes: """ from sqlalchemy.orm import relationship, backref +from sqlalchemy.ext import associationproxy from sqlalchemy import ( Column, ForeignKey, @@ -65,21 +66,23 @@ class ModelBase(object): return table_cls @classmethod - def foreign_key(cls, tablename, nullable=False): + def foreign_key(cls, table, nullable=False): """Return a ForeignKey object with the relevant - :param tablename: Unique id column in the parent table + :param table: Unique id column in the parent table :param nullable: Should the column be allowed to remain empty """ - table = cls._get_cls_by_tablename(tablename) - foreign_key_str = '{tablename}.{unique_id}'.format(tablename=tablename, + table = cls._get_cls_by_tablename(table.__tablename__) + foreign_key_str = '{tablename}.{unique_id}'.format(tablename=table.__tablename__, unique_id=table.storage_id_column()) - return Column(ForeignKey(foreign_key_str, ondelete='CASCADE'), nullable=nullable) + column = Column(ForeignKey(foreign_key_str, ondelete='CASCADE'), + nullable=nullable) + column.__remote_table_name = table.__name__ + return column @classmethod def one_to_many_relationship(cls, foreign_key_column, - parent_class, backreference=None): """Return a one-to-many SQL relationship object Meant to be used from inside the *child* object @@ -89,17 +92,18 @@ class ModelBase(object): :param foreign_key_column: The column of the foreign key (from the child table) :param backreference: The name to give to the reference to the child (on the parent table) """ - parent_table = cls._get_cls_by_tablename(parent_class) - primaryjoin_str = \ - '{parent_class_name}.{parent_unique_id} == {child_class.__name__}.{foreign_key_column}'\ + parent_table = cls._get_cls_by_tablename( + getattr(cls, foreign_key_column).__remote_table_name) + primaryjoin_str = '{parent_class_name}.{parent_unique_id} == ' \ + '{child_class.__name__}.{foreign_key_column}'\ .format( - parent_class_name=parent_class, + parent_class_name=parent_table.__name__, parent_unique_id=parent_table.storage_id_column(), child_class=cls, foreign_key_column=foreign_key_column ) return relationship( - parent_class, + parent_table.__name__, primaryjoin=primaryjoin_str, foreign_keys=[getattr(cls, foreign_key_column)], # The following line make sure that when the *parent* is @@ -123,29 +127,6 @@ class ModelBase(object): remote_side=remote_side_str, post_update=True) - -class ModelIdMixin(object): - id = Column(Integer, primary_key=True, autoincrement=True) - name = Column(Text, nullable=True, index=True) - - @classmethod - def storage_id_column(cls): - return 'id' - - @classmethod - def user_id_column(cls): - return 'name' - - -class ARIADeclarativeBase(object): - """ - Abstract base class for all SQL models that allows [de]serialization - """ - # This would be overridden once the models are created. Created for pylint. - __table__ = None - - _private_fields = [] - def to_dict(self, suppress_error=False): """Return a dict representation of the model @@ -155,7 +136,7 @@ class ARIADeclarativeBase(object): """ if suppress_error: res = dict() - for field in self.fields(): + for field in self.get_fields(): try: field_value = getattr(self, field) except AttributeError: @@ -168,12 +149,43 @@ class ARIADeclarativeBase(object): return res @classmethod + def _association_proxies(cls): + for col, value in cls.__table__.columns.items(): + if isinstance(value, associationproxy.AssociationProxy): + yield col + + @classmethod def fields(cls): """Return the list of field names for this table Mostly for backwards compatibility in the code (that uses `fields`) """ - return set(cls.__table__.columns.keys()) - set(cls._private_fields) + fields = set(cls._association_proxies()) + fields.update(cls.__table__.columns.keys()) + return fields - set(cls._private_fields) + + +class ModelIdMixin(object): + id = Column(Integer, primary_key=True, autoincrement=True) + name = Column(Text, nullable=True, index=True) + + @classmethod + def storage_id_column(cls): + return 'id' + + @classmethod + def user_id_column(cls): + return 'name' + + +class ARIADeclarativeBase(object): + """ + Abstract base class for all SQL models that allows [de]serialization + """ + # This would be overridden once the models are created. Created for pylint. + __table__ = None + + _private_fields = [] def __repr__(self): return '<{__class__.__name__} id=`{id}`>'.format( http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/ee420961/tests/orchestrator/context/test_toolbelt.py ---------------------------------------------------------------------- diff --git a/tests/orchestrator/context/test_toolbelt.py b/tests/orchestrator/context/test_toolbelt.py index 6f24ca5..b63811b 100644 --- a/tests/orchestrator/context/test_toolbelt.py +++ b/tests/orchestrator/context/test_toolbelt.py @@ -49,7 +49,7 @@ def executor(): def _get_elements(workflow_context): dependency_node = workflow_context.model.node.get_by_name(mock.models.DEPENDENCY_NODE_NAME) - dependency_node.host_fk = dependency_node.id + dependency_node.host = dependency_node workflow_context.model.node.update(dependency_node) dependency_node_instance = workflow_context.model.node_instance.get_by_name( http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/ee420961/tests/storage/test_models.py ---------------------------------------------------------------------- diff --git a/tests/storage/test_models.py b/tests/storage/test_models.py index 80f9fd3..fcf2c2d 100644 --- a/tests/storage/test_models.py +++ b/tests/storage/test_models.py @@ -194,15 +194,12 @@ class TestBlueprint(object): ) def test_blueprint_model_creation(self, empty_storage, is_valid, plan, description, created_at, updated_at, main_file_name): - if not is_valid: - with pytest.raises(exceptions.StorageError): - empty_storage.blueprint.put(Blueprint(plan=plan, description=description, - created_at=created_at, updated_at=updated_at, - main_file_name=main_file_name)) - else: - empty_storage.blueprint.put(Blueprint(plan=plan, description=description, - created_at=created_at, updated_at=updated_at, - main_file_name=main_file_name)) + _test_model(is_valid=is_valid, storage=empty_storage, model_name='blueprint', + model_cls=Blueprint, model_kwargs=dict(plan=plan, + description=description, + created_at=created_at, + updated_at=updated_at, + main_file_name=main_file_name)) class TestDeployment(object):
