[GitHub] incubator-ariatosca pull request #31: ARIA-30-SQL-based-storage-implementati...

[mxmrlv]

Github user mxmrlv commented on a diff in the pull request:

https://github.com/apache/incubator-ariatosca/pull/31#discussion_r90503715
  
--- Diff: aria/storage/mapi/sql.py ---
@@ -0,0 +1,368 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+SQLAlchemy based MAPI
+"""
+
+from sqlite3 import DatabaseError as SQLiteDBError
+from sqlalchemy.exc import SQLAlchemyError
+from sqlalchemy.sql.elements import Label
+
+try:
+from psycopg2 import DatabaseError as Psycopg2DBError
+sql_errors = (SQLAlchemyError, SQLiteDBError, Psycopg2DBError)
+except ImportError:
+sql_errors = (SQLAlchemyError, SQLiteDBError)
+Psycopg2DBError = None
+
+from aria.utils.collections import OrderedDict
+
+from ... import storage
+
+
+DEFAULT_SQL_DIALECT = 'sqlite'
+
+
+class SQLAlchemyModelAPI(storage.api.ModelAPI):
--- End diff --

get back on Sunday


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] incubator-ariatosca pull request #31: ARIA-30-SQL-based-storage-implementati...

[mxmrlv]

Github user mxmrlv commented on a diff in the pull request:

https://github.com/apache/incubator-ariatosca/pull/31#discussion_r90504139
  
--- Diff: aria/storage/models.py ---
@@ -60,353 +73,612 @@
 )
 
 # todo: sort this, maybe move from mgr or move from aria???
-ACTION_TYPES = ()
-ENTITY_TYPES = ()
+# TODO: this must change
+ACTION_TYPES = ('a')
+ENTITY_TYPES = ('b')
+
+
+def uuid_generator():
--- End diff --

remove


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] incubator-ariatosca pull request #31: ARIA-30-SQL-based-storage-implementati...

[mxmrlv]

Github user mxmrlv commented on a diff in the pull request:

https://github.com/apache/incubator-ariatosca/pull/31#discussion_r90504471
  
--- Diff: aria/storage/models.py ---
@@ -60,353 +73,612 @@
 )
 
 # todo: sort this, maybe move from mgr or move from aria???
-ACTION_TYPES = ()
-ENTITY_TYPES = ()
+# TODO: this must change
+ACTION_TYPES = ('a')
+ENTITY_TYPES = ('b')
+
+
+def uuid_generator():
+"""
+wrapper function which generates ids
+"""
+return str(uuid4())
 
 
-class Blueprint(Model):
+class Blueprint(SQLModelBase):
 """
-A Model which represents a blueprint
+Blueprint model representation.
 """
-plan = Field(type=dict)
-id = Field(type=basestring, default=uuid_generator)
-description = Field(type=(basestring, NoneType))
-created_at = Field(type=datetime)
-updated_at = Field(type=datetime)
-main_file_name = Field(type=basestring)
+__tablename__ = 'blueprints'
+
+storage_id = Column(Integer, primary_key=True, autoincrement=True)
+id = Column(Text, index=True)
 
+created_at = Column(DateTime, nullable=False, index=True)
--- End diff --

remove index


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] incubator-ariatosca pull request #31: ARIA-30-SQL-based-storage-implementati...

[mxmrlv]

Github user mxmrlv commented on a diff in the pull request:

https://github.com/apache/incubator-ariatosca/pull/31#discussion_r90503205
  
--- Diff: aria/storage/mapi/sql.py ---
@@ -0,0 +1,368 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+SQLAlchemy based MAPI
+"""
+
+from sqlite3 import DatabaseError as SQLiteDBError
+from sqlalchemy.exc import SQLAlchemyError
+from sqlalchemy.sql.elements import Label
+
+try:
+from psycopg2 import DatabaseError as Psycopg2DBError
+sql_errors = (SQLAlchemyError, SQLiteDBError, Psycopg2DBError)
--- End diff --

remove psycopg2


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] incubator-ariatosca pull request #31: ARIA-30-SQL-based-storage-implementati...

[mxmrlv]

Github user mxmrlv commented on a diff in the pull request:

https://github.com/apache/incubator-ariatosca/pull/31#discussion_r90504075
  
--- Diff: aria/storage/models.py ---
@@ -60,353 +73,612 @@
 )
 
 # todo: sort this, maybe move from mgr or move from aria???
-ACTION_TYPES = ()
-ENTITY_TYPES = ()
+# TODO: this must change
+ACTION_TYPES = ('a')
--- End diff --

move into deployment update step


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] incubator-ariatosca pull request #31: ARIA-30-SQL-based-storage-implementati...

[mxmrlv]

Github user mxmrlv commented on a diff in the pull request:

https://github.com/apache/incubator-ariatosca/pull/31#discussion_r90505762
  
--- Diff: aria/storage/models.py ---
@@ -60,353 +73,612 @@
 )
 
 # todo: sort this, maybe move from mgr or move from aria???
-ACTION_TYPES = ()
-ENTITY_TYPES = ()
+# TODO: this must change
+ACTION_TYPES = ('a')
+ENTITY_TYPES = ('b')
+
+
+def uuid_generator():
+"""
+wrapper function which generates ids
+"""
+return str(uuid4())
 
 
-class Blueprint(Model):
+class Blueprint(SQLModelBase):
 """
-A Model which represents a blueprint
+Blueprint model representation.
 """
-plan = Field(type=dict)
-id = Field(type=basestring, default=uuid_generator)
-description = Field(type=(basestring, NoneType))
-created_at = Field(type=datetime)
-updated_at = Field(type=datetime)
-main_file_name = Field(type=basestring)
+__tablename__ = 'blueprints'
+
+storage_id = Column(Integer, primary_key=True, autoincrement=True)
+id = Column(Text, index=True)
 
+created_at = Column(DateTime, nullable=False, index=True)
+main_file_name = Column(Text, nullable=False)
+plan = Column(MutableDict.as_mutable(Dict), nullable=False)
+updated_at = Column(DateTime)
+description = Column(Text)
 
-class Snapshot(Model):
+
+class Snapshot(SQLModelBase):
 """
-A Model which represents a snapshot
+Snapshot model representation.
 """
+__tablename__ = 'snapshots'
+
 CREATED = 'created'
 FAILED = 'failed'
 CREATING = 'creating'
 UPLOADED = 'uploaded'
+
+STATES = [CREATED, FAILED, CREATING, UPLOADED]
 END_STATES = [CREATED, FAILED, UPLOADED]
 
-id = Field(type=basestring, default=uuid_generator)
-created_at = Field(type=datetime)
-status = Field(type=basestring)
-error = Field(type=basestring, default=None)
+storage_id = Column(Integer, primary_key=True, autoincrement=True)
+id = Column(Text, index=True)
 
+created_at = Column(DateTime, nullable=False, index=True)
+status = Column(Enum(*STATES, name='snapshot_status'))
+error = Column(Text)
 
-class Deployment(Model):
-"""
-A Model which represents a deployment
-"""
-id = Field(type=basestring, default=uuid_generator)
-description = Field(type=(basestring, NoneType))
-created_at = Field(type=datetime)
-updated_at = Field(type=datetime)
-blueprint_id = Field(type=basestring)
-workflows = Field(type=dict)
-inputs = Field(type=dict, default=lambda: {})
-policy_types = Field(type=dict, default=lambda: {})
-policy_triggers = Field(type=dict, default=lambda: {})
-groups = Field(type=dict, default=lambda: {})
-outputs = Field(type=dict, default=lambda: {})
-scaling_groups = Field(type=dict, default=lambda: {})
-
-
-class DeploymentUpdateStep(Model):
+
+class Deployment(SQLModelBase):
 """
-A Model which represents a deployment update step
+Deployment model representation.
 """
-id = Field(type=basestring, default=uuid_generator)
-action = Field(type=basestring, choices=ACTION_TYPES)
-entity_type = Field(type=basestring, choices=ENTITY_TYPES)
-entity_id = Field(type=basestring)
-supported = Field(type=bool, default=True)
-
-def __hash__(self):
-return hash((self.id, self.entity_id))
+__tablename__ = 'deployments'
+
+# See base class for an explanation on these properties
+join_properties = {
+'blueprint_id': {
+# No need to provide the Blueprint table, as it's already 
joined
+'models': [Blueprint],
+'column': Blueprint.id.label('blueprint_id')
+},
+}
+join_order = 2
+
+_private_fields = ['blueprint_storage_id']
--- End diff --

add comment in base


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] incubator-ariatosca pull request #31: ARIA-30-SQL-based-storage-implementati...

[mxmrlv]

Github user mxmrlv commented on a diff in the pull request:

https://github.com/apache/incubator-ariatosca/pull/31#discussion_r90502146
  
--- Diff: aria/storage/filesystem_api.py ---
@@ -0,0 +1,39 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+Filesystem based API Base
+"""
+from multiprocessing import RLock
--- End diff --

move to filesystem rapi


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] incubator-ariatosca pull request #31: ARIA-30-SQL-based-storage-implementati...

[mxmrlv]

Github user mxmrlv commented on a diff in the pull request:

https://github.com/apache/incubator-ariatosca/pull/31#discussion_r90501806
  
--- Diff: aria/storage/api.py ---
@@ -0,0 +1,219 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+General storage API
+"""
+from contextlib import contextmanager
+
+from . import exceptions
+
+
+class StorageAPI(object):
+"""
+General storage Base API
+"""
+def create(self, **kwargs):
+"""
+Create a storage API.
+:param kwargs:
+:return:
+"""
+raise NotImplementedError('Subclass must implement abstract create 
method')
+
+@contextmanager
+def connect(self):
+"""
+Established a connection and destroys it after use.
+:return:
+"""
+try:
+self._establish_connection()
+yield self
+except BaseException as e:
+raise exceptions.StorageError(str(e))
+finally:
+self._destroy_connection()
+
+def _establish_connection(self):
+"""
+Establish a conenction. used in the 'connect' contextmanager.
+:return:
+"""
+pass
+
+def _destroy_connection(self):
+"""
+Destroy a connection. used in the 'connect' contextmanager.
+:return:
+"""
+pass
+
+def __getattr__(self, item):
+try:
+return self.registered[item]
+except KeyError:
+return super(StorageAPI, self).__getattribute__(item)
+
+
+class ModelAPI(StorageAPI):
+"""
+A Base object for the model.
+"""
+def __init__(self, model_cls, name=None, **kwargs):
+"""
+Base model API
+
+:param model_cls: the representing class of the model
+:param str name: the name of the model
+:param kwargs:
+"""
+super(ModelAPI, self).__init__(**kwargs)
+self._model_cls = model_cls
+self._name = name or generate_lower_name(model_cls)
+
+@property
+def name(self):
+"""
+The name of the class
+:return: name of the class
+"""
+return self._name
+
+@property
+def model_cls(self):
+"""
+The class represting the model
+:return:
+"""
+return self._model_cls
+
+def get(self, entry_id, filters=None, **kwargs):
+"""
+Get entry from storage.
+
+:param entry_id:
+:param kwargs:
+:return:
+"""
+raise NotImplementedError('Subclass must implement abstract get 
method')
+
+def store(self, entry, **kwargs):
+"""
+Store entry in storage
+
+:param entry:
+:param kwargs:
+:return:
+"""
+raise NotImplementedError('Subclass must implement abstract store 
method')
+
+def delete(self, entry_id, **kwargs):
+"""
+Delete entry from storage.
+
+:param entry_id:
+:param kwargs:
+:return:
+"""
+raise NotImplementedError('Subclass must implement abstract delete 
method')
+
+def __iter__(self):
+return self.iter()
+
+def iter(self, **kwargs):
+"""
+Iter over the entries in storage.
+
+:param kwargs:
+:return:
+"""
+raise NotImplementedError('Subclass must implement abstract iter 
method')
+
+def update(self, entry, **kwargs):
+"""
+Update entry in storage.
+
+:param entry:
+:param kwargs:
+:return:
+"""
+raise NotImplementedError('Subclass must implement abstract 

[GitHub] incubator-ariatosca pull request #31: ARIA-30-SQL-based-storage-implementati...

[mxmrlv]

Github user mxmrlv commented on a diff in the pull request:

https://github.com/apache/incubator-ariatosca/pull/31#discussion_r90502764
  
--- Diff: aria/storage/mapi/inmemory.py ---
@@ -0,0 +1,148 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
--- End diff --

delete


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] incubator-ariatosca pull request #31: ARIA-30-SQL-based-storage-implementati...

[mxmrlv]

Github user mxmrlv commented on a diff in the pull request:

https://github.com/apache/incubator-ariatosca/pull/31#discussion_r90503314
  
--- Diff: aria/storage/mapi/sql.py ---
@@ -0,0 +1,368 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+SQLAlchemy based MAPI
+"""
+
+from sqlite3 import DatabaseError as SQLiteDBError
+from sqlalchemy.exc import SQLAlchemyError
+from sqlalchemy.sql.elements import Label
+
+try:
+from psycopg2 import DatabaseError as Psycopg2DBError
+sql_errors = (SQLAlchemyError, SQLiteDBError, Psycopg2DBError)
--- End diff --

check why all these errors are imported


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] incubator-ariatosca pull request #31: ARIA-30-SQL-based-storage-implementati...

[mxmrlv]

Github user mxmrlv commented on a diff in the pull request:

https://github.com/apache/incubator-ariatosca/pull/31#discussion_r90499710
  
--- Diff: aria/storage/__init__.py ---
@@ -37,354 +37,93 @@
 * drivers - module, a pool of Aria standard drivers.
 * StorageDriver - class, abstract model implementation.
 """
-# todo: rewrite the above package documentation
-# (something like explaning the two types of storage - models and 
resources)
 
-from collections import namedtuple
-
-from .structures import Storage, Field, Model, IterField, PointerField
-from .drivers import (
-ModelDriver,
-ResourceDriver,
-FileSystemResourceDriver,
-FileSystemModelDriver,
+from aria.logger import LoggerMixin
+from . import (
+models,
+exceptions,
+api as storage_api,
+structures
 )
-from . import models, exceptions
+
 
 __all__ = (
 'ModelStorage',
-'ResourceStorage',
-'FileSystemModelDriver',
 'models',
 'structures',
-'Field',
-'IterField',
-'PointerField',
-'Model',
-'drivers',
-'ModelDriver',
-'ResourceDriver',
-'FileSystemResourceDriver',
 )
-# todo: think about package output api's...
-# todo: in all drivers name => entry_type
-# todo: change in documentation str => basestring
 
 
-class ModelStorage(Storage):
+class Storage(LoggerMixin):
 """
-Managing the models storage.
+Represents the storage
 """
-def __init__(self, driver, model_classes=(), **kwargs):
-"""
-Simple storage client api for Aria applications.
-The storage instance defines the tables/documents/code api.
-
-:param ModelDriver driver: model storage driver.
-:param model_classes: the models to register.
-"""
-assert isinstance(driver, ModelDriver)
-super(ModelStorage, self).__init__(driver, model_classes, **kwargs)
-
-def __getattr__(self, table):
-"""
-getattr is a shortcut to simple api
-
-for Example:
->> storage = ModelStorage(driver=FileSystemModelDriver('/tmp'))
->> node_table = storage.node
->> for node in node_table:
->> print node
-
-:param str table: table name to get
-:return: a storage object that mapped to the table name
-"""
-return super(ModelStorage, self).__getattr__(table)
-
-def register(self, model_cls):
-"""
-Registers the model type in the resource storage manager.
-:param model_cls: the model to register.
-"""
-model_name = generate_lower_name(model_cls)
-model_api = _ModelApi(model_name, self.driver, model_cls)
-self.registered[model_name] = model_api
-
-for pointer_schema_register in model_api.pointer_mapping.values():
-model_cls = pointer_schema_register.model_cls
-self.register(model_cls)
-
-_Pointer = namedtuple('_Pointer', 'name, is_iter')
-
-
-class _ModelApi(object):
-def __init__(self, name, driver, model_cls):
-"""
-Managing the model in the storage, using the driver.
-
-:param basestring name: the name of the model.
-:param ModelDriver driver: the driver which supports this model in 
the storage.
-:param Model model_cls: table/document class model.
-"""
-assert isinstance(driver, ModelDriver)
-assert issubclass(model_cls, Model)
-self.name = name
-self.driver = driver
-self.model_cls = model_cls
-self.pointer_mapping = {}
-self._setup_pointers_mapping()
-
-def _setup_pointers_mapping(self):
-for field_name, field_cls in vars(self.model_cls).items():
-if not(isinstance(field_cls, PointerField) and field_cls.type):
-continue
-pointer_key = _Pointer(field_name, 
is_iter=isinstance(field_cls, IterField))
-self.pointer_mapping[pointer_key] = self.__class__(
-name=generate_lower_name(field_cls.type),
-driver=self.driver,
-model_cls=field_cls.type)
-
-def __iter__(self):
-return self.iter()
+def __init__(self, api, items=(), api_params=None, **kwargs):
+self._api_params = api_params or {}
+super(Storage, self).__init__(**kwargs)
+self.api = api
+self.registered = {}
+for item in items:
+self.register(item)
+self.logger.debug('{name} object is ready: 

[GitHub] incubator-ariatosca pull request #31: ARIA-30-SQL-based-storage-implementati...

[mxmrlv]

Github user mxmrlv commented on a diff in the pull request:

https://github.com/apache/incubator-ariatosca/pull/31#discussion_r90496245
  
--- Diff: aria/orchestrator/context/workflow.py ---
@@ -64,19 +66,27 @@ def nodes(self):
 """
 Iterator over nodes
 """
-return self.model.node.iter(filters={'blueprint_id': 
self.blueprint.id})
+return self.model.node.iter(
+filters={
+'deployment_storage_id': self.deployment.storage_id
--- End diff --

create an issue considering unique id for different models


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] incubator-ariatosca pull request #31: ARIA-30-SQL-based-storage-implementati...

[mxmrlv]

Github user mxmrlv commented on a diff in the pull request:

https://github.com/apache/incubator-ariatosca/pull/31#discussion_r90501110
  
--- Diff: aria/storage/api.py ---
@@ -0,0 +1,219 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+General storage API
+"""
+from contextlib import contextmanager
+
+from . import exceptions
+
+
+class StorageAPI(object):
+"""
+General storage Base API
+"""
+def create(self, **kwargs):
+"""
+Create a storage API.
+:param kwargs:
+:return:
+"""
+raise NotImplementedError('Subclass must implement abstract create 
method')
+
+@contextmanager
+def connect(self):
+"""
+Established a connection and destroys it after use.
+:return:
+"""
+try:
+self._establish_connection()
+yield self
+except BaseException as e:
+raise exceptions.StorageError(str(e))
+finally:
+self._destroy_connection()
+
+def _establish_connection(self):
+"""
+Establish a conenction. used in the 'connect' contextmanager.
+:return:
+"""
+pass
+
+def _destroy_connection(self):
+"""
+Destroy a connection. used in the 'connect' contextmanager.
+:return:
+"""
+pass
+
+def __getattr__(self, item):
--- End diff --

remove


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] incubator-ariatosca pull request #31: ARIA-30-SQL-based-storage-implementati...

[mxmrlv]

Github user mxmrlv commented on a diff in the pull request:

https://github.com/apache/incubator-ariatosca/pull/31#discussion_r90501812
  
--- Diff: aria/storage/api.py ---
@@ -0,0 +1,219 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+General storage API
+"""
+from contextlib import contextmanager
+
+from . import exceptions
+
+
+class StorageAPI(object):
+"""
+General storage Base API
+"""
+def create(self, **kwargs):
+"""
+Create a storage API.
+:param kwargs:
+:return:
+"""
+raise NotImplementedError('Subclass must implement abstract create 
method')
+
+@contextmanager
+def connect(self):
+"""
+Established a connection and destroys it after use.
+:return:
+"""
+try:
+self._establish_connection()
+yield self
+except BaseException as e:
+raise exceptions.StorageError(str(e))
+finally:
+self._destroy_connection()
+
+def _establish_connection(self):
+"""
+Establish a conenction. used in the 'connect' contextmanager.
+:return:
+"""
+pass
+
+def _destroy_connection(self):
+"""
+Destroy a connection. used in the 'connect' contextmanager.
+:return:
+"""
+pass
+
+def __getattr__(self, item):
+try:
+return self.registered[item]
+except KeyError:
+return super(StorageAPI, self).__getattribute__(item)
+
+
+class ModelAPI(StorageAPI):
+"""
+A Base object for the model.
+"""
+def __init__(self, model_cls, name=None, **kwargs):
+"""
+Base model API
+
+:param model_cls: the representing class of the model
+:param str name: the name of the model
+:param kwargs:
+"""
+super(ModelAPI, self).__init__(**kwargs)
+self._model_cls = model_cls
+self._name = name or generate_lower_name(model_cls)
+
+@property
+def name(self):
+"""
+The name of the class
+:return: name of the class
+"""
+return self._name
+
+@property
+def model_cls(self):
+"""
+The class represting the model
+:return:
+"""
+return self._model_cls
+
+def get(self, entry_id, filters=None, **kwargs):
+"""
+Get entry from storage.
+
+:param entry_id:
+:param kwargs:
+:return:
+"""
+raise NotImplementedError('Subclass must implement abstract get 
method')
+
+def store(self, entry, **kwargs):
+"""
+Store entry in storage
+
+:param entry:
+:param kwargs:
+:return:
+"""
+raise NotImplementedError('Subclass must implement abstract store 
method')
+
+def delete(self, entry_id, **kwargs):
+"""
+Delete entry from storage.
+
+:param entry_id:
+:param kwargs:
+:return:
+"""
+raise NotImplementedError('Subclass must implement abstract delete 
method')
+
+def __iter__(self):
+return self.iter()
+
+def iter(self, **kwargs):
+"""
+Iter over the entries in storage.
+
+:param kwargs:
+:return:
+"""
+raise NotImplementedError('Subclass must implement abstract iter 
method')
+
+def update(self, entry, **kwargs):
+"""
+Update entry in storage.
+
+:param entry:
+:param kwargs:
+:return:
+"""
+raise NotImplementedError('Subclass must implement abstract 

[GitHub] incubator-ariatosca pull request #31: ARIA-30-SQL-based-storage-implementati...

[mxmrlv]

Github user mxmrlv commented on a diff in the pull request:

https://github.com/apache/incubator-ariatosca/pull/31#discussion_r90498171
  
--- Diff: aria/storage/__init__.py ---
@@ -37,354 +37,93 @@
 * drivers - module, a pool of Aria standard drivers.
 * StorageDriver - class, abstract model implementation.
 """
-# todo: rewrite the above package documentation
-# (something like explaning the two types of storage - models and 
resources)
 
-from collections import namedtuple
-
-from .structures import Storage, Field, Model, IterField, PointerField
-from .drivers import (
-ModelDriver,
-ResourceDriver,
-FileSystemResourceDriver,
-FileSystemModelDriver,
+from aria.logger import LoggerMixin
+from . import (
+models,
+exceptions,
+api as storage_api,
+structures
 )
-from . import models, exceptions
+
 
 __all__ = (
 'ModelStorage',
-'ResourceStorage',
-'FileSystemModelDriver',
 'models',
 'structures',
-'Field',
-'IterField',
-'PointerField',
-'Model',
-'drivers',
-'ModelDriver',
-'ResourceDriver',
-'FileSystemResourceDriver',
 )
-# todo: think about package output api's...
-# todo: in all drivers name => entry_type
-# todo: change in documentation str => basestring
 
 
-class ModelStorage(Storage):
+class Storage(LoggerMixin):
 """
-Managing the models storage.
+Represents the storage
 """
-def __init__(self, driver, model_classes=(), **kwargs):
-"""
-Simple storage client api for Aria applications.
-The storage instance defines the tables/documents/code api.
-
-:param ModelDriver driver: model storage driver.
-:param model_classes: the models to register.
-"""
-assert isinstance(driver, ModelDriver)
-super(ModelStorage, self).__init__(driver, model_classes, **kwargs)
-
-def __getattr__(self, table):
-"""
-getattr is a shortcut to simple api
-
-for Example:
->> storage = ModelStorage(driver=FileSystemModelDriver('/tmp'))
->> node_table = storage.node
->> for node in node_table:
->> print node
-
-:param str table: table name to get
-:return: a storage object that mapped to the table name
-"""
-return super(ModelStorage, self).__getattr__(table)
-
-def register(self, model_cls):
-"""
-Registers the model type in the resource storage manager.
-:param model_cls: the model to register.
-"""
-model_name = generate_lower_name(model_cls)
-model_api = _ModelApi(model_name, self.driver, model_cls)
-self.registered[model_name] = model_api
-
-for pointer_schema_register in model_api.pointer_mapping.values():
-model_cls = pointer_schema_register.model_cls
-self.register(model_cls)
-
-_Pointer = namedtuple('_Pointer', 'name, is_iter')
-
-
-class _ModelApi(object):
-def __init__(self, name, driver, model_cls):
-"""
-Managing the model in the storage, using the driver.
-
-:param basestring name: the name of the model.
-:param ModelDriver driver: the driver which supports this model in 
the storage.
-:param Model model_cls: table/document class model.
-"""
-assert isinstance(driver, ModelDriver)
-assert issubclass(model_cls, Model)
-self.name = name
-self.driver = driver
-self.model_cls = model_cls
-self.pointer_mapping = {}
-self._setup_pointers_mapping()
-
-def _setup_pointers_mapping(self):
-for field_name, field_cls in vars(self.model_cls).items():
-if not(isinstance(field_cls, PointerField) and field_cls.type):
-continue
-pointer_key = _Pointer(field_name, 
is_iter=isinstance(field_cls, IterField))
-self.pointer_mapping[pointer_key] = self.__class__(
-name=generate_lower_name(field_cls.type),
-driver=self.driver,
-model_cls=field_cls.type)
-
-def __iter__(self):
-return self.iter()
+def __init__(self, api, items=(), api_params=None, **kwargs):
--- End diff --

api_cls, api_params, items


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact 

[GitHub] incubator-ariatosca pull request #31: ARIA-30-SQL-based-storage-implementati...

[mxmrlv]

Github user mxmrlv commented on a diff in the pull request:

https://github.com/apache/incubator-ariatosca/pull/31#discussion_r90495335
  
--- Diff: aria/orchestrator/context/workflow.py ---
@@ -64,19 +66,27 @@ def nodes(self):
 """
 Iterator over nodes
 """
-return self.model.node.iter(filters={'blueprint_id': 
self.blueprint.id})
+return self.model.node.iter(
+filters={
+'deployment_storage_id': self.deployment.storage_id
--- End diff --

storage_id -> id
id -> name (not unique)


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] incubator-ariatosca pull request #31: ARIA-30-SQL-based-storage-implementati...

[mxmrlv]

Github user mxmrlv commented on a diff in the pull request:

https://github.com/apache/incubator-ariatosca/pull/31#discussion_r90502574
  
--- Diff: aria/storage/mapi/filesystem.py ---
@@ -0,0 +1,118 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
--- End diff --

delete


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] incubator-ariatosca pull request #31: ARIA-30-SQL-based-storage-implementati...

[mxmrlv]

Github user mxmrlv commented on a diff in the pull request:

https://github.com/apache/incubator-ariatosca/pull/31#discussion_r90499868
  
--- Diff: aria/storage/__init__.py ---
@@ -37,354 +37,93 @@
 * drivers - module, a pool of Aria standard drivers.
 * StorageDriver - class, abstract model implementation.
 """
-# todo: rewrite the above package documentation
-# (something like explaning the two types of storage - models and 
resources)
 
-from collections import namedtuple
-
-from .structures import Storage, Field, Model, IterField, PointerField
-from .drivers import (
-ModelDriver,
-ResourceDriver,
-FileSystemResourceDriver,
-FileSystemModelDriver,
+from aria.logger import LoggerMixin
+from . import (
+models,
+exceptions,
+api as storage_api,
+structures
 )
-from . import models, exceptions
+
 
 __all__ = (
 'ModelStorage',
-'ResourceStorage',
-'FileSystemModelDriver',
 'models',
 'structures',
-'Field',
-'IterField',
-'PointerField',
-'Model',
-'drivers',
-'ModelDriver',
-'ResourceDriver',
-'FileSystemResourceDriver',
 )
-# todo: think about package output api's...
-# todo: in all drivers name => entry_type
-# todo: change in documentation str => basestring
 
 
-class ModelStorage(Storage):
+class Storage(LoggerMixin):
 """
-Managing the models storage.
+Represents the storage
 """
-def __init__(self, driver, model_classes=(), **kwargs):
-"""
-Simple storage client api for Aria applications.
-The storage instance defines the tables/documents/code api.
-
-:param ModelDriver driver: model storage driver.
-:param model_classes: the models to register.
-"""
-assert isinstance(driver, ModelDriver)
-super(ModelStorage, self).__init__(driver, model_classes, **kwargs)
-
-def __getattr__(self, table):
-"""
-getattr is a shortcut to simple api
-
-for Example:
->> storage = ModelStorage(driver=FileSystemModelDriver('/tmp'))
->> node_table = storage.node
->> for node in node_table:
->> print node
-
-:param str table: table name to get
-:return: a storage object that mapped to the table name
-"""
-return super(ModelStorage, self).__getattr__(table)
-
-def register(self, model_cls):
-"""
-Registers the model type in the resource storage manager.
-:param model_cls: the model to register.
-"""
-model_name = generate_lower_name(model_cls)
-model_api = _ModelApi(model_name, self.driver, model_cls)
-self.registered[model_name] = model_api
-
-for pointer_schema_register in model_api.pointer_mapping.values():
-model_cls = pointer_schema_register.model_cls
-self.register(model_cls)
-
-_Pointer = namedtuple('_Pointer', 'name, is_iter')
-
-
-class _ModelApi(object):
-def __init__(self, name, driver, model_cls):
-"""
-Managing the model in the storage, using the driver.
-
-:param basestring name: the name of the model.
-:param ModelDriver driver: the driver which supports this model in 
the storage.
-:param Model model_cls: table/document class model.
-"""
-assert isinstance(driver, ModelDriver)
-assert issubclass(model_cls, Model)
-self.name = name
-self.driver = driver
-self.model_cls = model_cls
-self.pointer_mapping = {}
-self._setup_pointers_mapping()
-
-def _setup_pointers_mapping(self):
-for field_name, field_cls in vars(self.model_cls).items():
-if not(isinstance(field_cls, PointerField) and field_cls.type):
-continue
-pointer_key = _Pointer(field_name, 
is_iter=isinstance(field_cls, IterField))
-self.pointer_mapping[pointer_key] = self.__class__(
-name=generate_lower_name(field_cls.type),
-driver=self.driver,
-model_cls=field_cls.type)
-
-def __iter__(self):
-return self.iter()
+def __init__(self, api, items=(), api_params=None, **kwargs):
+self._api_params = api_params or {}
+super(Storage, self).__init__(**kwargs)
+self.api = api
+self.registered = {}
+for item in items:
+self.register(item)
+self.logger.debug('{name} object is ready: 

[GitHub] incubator-ariatosca pull request #31: ARIA-30-SQL-based-storage-implementati...

[mxmrlv]

Github user mxmrlv commented on a diff in the pull request:

https://github.com/apache/incubator-ariatosca/pull/31#discussion_r90500929
  
--- Diff: aria/storage/api.py ---
@@ -0,0 +1,219 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+General storage API
+"""
+from contextlib import contextmanager
+
+from . import exceptions
+
+
+class StorageAPI(object):
+"""
+General storage Base API
+"""
+def create(self, **kwargs):
+"""
+Create a storage API.
+:param kwargs:
+:return:
+"""
+raise NotImplementedError('Subclass must implement abstract create 
method')
+
+@contextmanager
+def connect(self):
--- End diff --

consider removing


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[jira] [Created] (ARIATOSCA-36) Plugin working directory storage

[Ran Ziv (JIRA)]

Ran Ziv created ARIATOSCA-36:


 Summary: Plugin working directory storage
 Key: ARIATOSCA-36
 URL: https://issues.apache.org/jira/browse/ARIATOSCA-36
 Project: AriaTosca
  Issue Type: Story
Reporter: Ran Ziv


A plugin should have its own storage space, available via the plugin model 
object



--
This message was sent by Atlassian JIRA
(v6.3.4#6332)

[jira] [Created] (ARIATOSCA-35) Add default logger in operations

[Ran Ziv (JIRA)]

Ran Ziv created ARIATOSCA-35:


 Summary: Add default logger in operations
 Key: ARIATOSCA-35
 URL: https://issues.apache.org/jira/browse/ARIATOSCA-35
 Project: AriaTosca
  Issue Type: Story
Reporter: Ran Ziv






--
This message was sent by Atlassian JIRA
(v6.3.4#6332)

[jira] [Created] (ARIATOSCA-34) API for rendering a downloaded resource

[Ran Ziv (JIRA)]

Ran Ziv created ARIATOSCA-34:


 Summary: API for rendering a downloaded resource
 Key: ARIATOSCA-34
 URL: https://issues.apache.org/jira/browse/ARIATOSCA-34
 Project: AriaTosca
  Issue Type: Story
Reporter: Ran Ziv


Add an API for downloading + rendering a resource from the storage, similar to 
the workflow-context's "download_resource" method



--
This message was sent by Atlassian JIRA
(v6.3.4#6332)

[jira] [Created] (ARIATOSCA-32) Implement scale workflow

[Ran Ziv (JIRA)]

Ran Ziv created ARIATOSCA-32:


 Summary: Implement scale workflow
 Key: ARIATOSCA-32
 URL: https://issues.apache.org/jira/browse/ARIATOSCA-32
 Project: AriaTosca
  Issue Type: Story
Reporter: Ran Ziv






--
This message was sent by Atlassian JIRA
(v6.3.4#6332)

[jira] [Created] (ARIATOSCA-31) Investigate consolidation of extensions mechanisms

[Ran Ziv (JIRA)]

Ran Ziv created ARIATOSCA-31:


 Summary: Investigate consolidation of extensions mechanisms
 Key: ARIATOSCA-31
 URL: https://issues.apache.org/jira/browse/ARIATOSCA-31
 Project: AriaTosca
  Issue Type: Task
Reporter: Ran Ziv
Priority: Critical






--
This message was sent by Atlassian JIRA
(v6.3.4#6332)

[2/3] incubator-ariatosca git commit: ARIA-30 SQL based storage implementation

[mxmrlv]

http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/c9df5b14/aria/storage/mapi/sql.py
--
diff --git a/aria/storage/mapi/sql.py b/aria/storage/mapi/sql.py
new file mode 100644
index 000..652dc9f
--- /dev/null
+++ b/aria/storage/mapi/sql.py
@@ -0,0 +1,368 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+SQLAlchemy based MAPI
+"""
+
+from sqlite3 import DatabaseError as SQLiteDBError
+from sqlalchemy.exc import SQLAlchemyError
+from sqlalchemy.sql.elements import Label
+
+try:
+from psycopg2 import DatabaseError as Psycopg2DBError
+sql_errors = (SQLAlchemyError, SQLiteDBError, Psycopg2DBError)
+except ImportError:
+sql_errors = (SQLAlchemyError, SQLiteDBError)
+Psycopg2DBError = None
+
+from aria.utils.collections import OrderedDict
+
+from ... import storage
+
+
+DEFAULT_SQL_DIALECT = 'sqlite'
+
+
+class SQLAlchemyModelAPI(storage.api.ModelAPI):
+"""
+SQL based MAPI.
+"""
+
+def __init__(self,
+ engine,
+ session,
+ **kwargs):
+super(SQLAlchemyModelAPI, self).__init__(**kwargs)
+self._engine = engine
+self._session = session
+
+def get(self, entry_id, include=None, filters=None, locking=False, 
**kwargs):
+"""Return a single result based on the model class and element ID
+"""
+filters = filters or {'id': entry_id}
+query = self._get_query(include, filters)
+if locking:
+query = query.with_for_update()
+result = query.first()
+
+if not result:
+raise storage.exceptions.StorageError(
+'Requested {0} with ID `{1}` was not found'
+.format(self.model_cls.__name__, entry_id)
+)
+return result
+
+def iter(self,
+ include=None,
+ filters=None,
+ pagination=None,
+ sort=None,
+ **kwargs):
+"""Return a (possibly empty) list of `model_class` results
+"""
+query = self._get_query(include, filters, sort)
+
+results, _, _, _ = self._paginate(query, pagination)
+
+for result in results:
+yield result
+
+def store(self, entry, **kwargs):
+"""Create a `model_class` instance from a serializable `model` object
+
+:param entry: A dict with relevant kwargs, or an instance of a class
+that has a `to_dict` method, and whose attributes match the columns
+of `model_class` (might also my just an instance of `model_class`)
+:return: An instance of `model_class`
+"""
+self._session.add(entry)
+self._safe_commit()
+return entry
+
+def delete(self, entry_id, filters=None, **kwargs):
+"""Delete a single result based on the model class and element ID
+"""
+try:
+instance = self.get(
+entry_id,
+filters=filters
+)
+except storage.exceptions.StorageError:
+raise storage.exceptions.StorageError(
+'Could not delete {0} with ID `{1}` - element not found'
+.format(
+self.model_cls.__name__,
+entry_id
+)
+)
+self._load_properties(instance)
+self._session.delete(instance)
+self._safe_commit()
+return instance
+
+# TODO: this might need rework
+def update(self, entry, **kwargs):
+"""Add `instance` to the DB session, and attempt to commit
+
+:return: The updated instance
+"""
+return self.store(entry)
+
+def refresh(self, entry):
+"""Reload the instance with fresh information from the DB
+
+:param entry: Instance to be re-loaded from the DB
+:return: The refreshed instance
+"""
+self._session.refresh(entry)
+self._load_properties(entry)
+return entry
+
+def _destroy_connection(self):
+pass
+
+def _establish_connection(self):
+pass
+
+def create(self):
+self.model_cls.__table__.create(self._engine)
+
+def drop(self):
+"""
+

[3/3] incubator-ariatosca git commit: ARIA-30 SQL based storage implementation

[mxmrlv]

ARIA-30 SQL based storage implementation


Project: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/commit/c9df5b14
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/tree/c9df5b14
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/diff/c9df5b14

Branch: refs/heads/ARIA-30-SQL-based-storage-implementation
Commit: c9df5b140498ca907d0f5b6ebca6cdd2163bcfcd
Parents: fe974e4
Author: mxmrlv 
Authored: Sun Nov 27 13:20:46 2016 +0200
Committer: mxmrlv 
Committed: Thu Dec 1 15:52:25 2016 +0200

--
 aria/__init__.py|  43 +-
 aria/orchestrator/__init__.py   |   4 +-
 aria/orchestrator/context/common.py |   2 +-
 aria/orchestrator/context/exceptions.py |   4 +-
 aria/orchestrator/context/operation.py  |   8 +-
 aria/orchestrator/context/toolbelt.py   |  13 +-
 aria/orchestrator/context/workflow.py   |  20 +-
 aria/orchestrator/exceptions.py |   7 +-
 aria/orchestrator/workflows/api/task.py |  10 +-
 aria/orchestrator/workflows/builtin/heal.py |  25 +-
 aria/orchestrator/workflows/builtin/install.py  |   7 +-
 .../orchestrator/workflows/builtin/uninstall.py |   7 +-
 .../orchestrator/workflows/builtin/workflows.py |  13 +-
 aria/orchestrator/workflows/core/task.py|  21 +-
 aria/storage/__init__.py| 379 ++--
 aria/storage/api.py | 219 +
 aria/storage/drivers.py | 416 -
 aria/storage/exceptions.py  |   4 +-
 aria/storage/filesystem_api.py  |  39 +
 aria/storage/mapi/__init__.py   |  20 +
 aria/storage/mapi/filesystem.py | 118 +++
 aria/storage/mapi/inmemory.py   | 148 +++
 aria/storage/mapi/sql.py| 368 
 aria/storage/models.py  | 912 +--
 aria/storage/rapi/__init__.py   |  18 +
 aria/storage/rapi/filesystem.py | 119 +++
 aria/storage/structures.py  | 402 
 requirements.txt|   1 +
 tests/mock/context.py   |  50 +-
 tests/mock/models.py|  68 +-
 tests/orchestrator/context/test_operation.py|  36 +-
 tests/orchestrator/context/test_toolbelt.py |  47 +-
 tests/orchestrator/context/test_workflow.py |  10 +-
 tests/orchestrator/workflows/api/test_task.py   |  68 +-
 .../orchestrator/workflows/builtin/__init__.py  |  35 +-
 .../workflows/builtin/test_execute_operation.py |  11 +-
 .../orchestrator/workflows/builtin/test_heal.py |  18 +-
 .../workflows/builtin/test_install.py   |  14 +-
 .../workflows/builtin/test_uninstall.py |  12 +-
 .../orchestrator/workflows/core/test_engine.py  |  71 +-
 tests/orchestrator/workflows/core/test_task.py  |  20 +-
 .../test_task_graph_into_exececution_graph.py   |  10 +-
 tests/requirements.txt  |   2 +-
 tests/storage/__init__.py   |  38 +-
 tests/storage/test_drivers.py   | 135 ---
 tests/storage/test_field.py | 124 ---
 tests/storage/test_model_storage.py | 167 ++--
 tests/storage/test_models.py| 364 
 tests/storage/test_models_api.py|  70 --
 tests/storage/test_resource_storage.py  |  57 +-
 50 files changed, 2296 insertions(+), 2478 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/c9df5b14/aria/__init__.py
--
diff --git a/aria/__init__.py b/aria/__init__.py
index 3f81f98..6e810f0 100644
--- a/aria/__init__.py
+++ b/aria/__init__.py
@@ -23,7 +23,6 @@ import pkgutil
 from .VERSION import version as __version__
 
 from .orchestrator.decorators import workflow, operation
-from .storage import ModelStorage, ResourceStorage, models, ModelDriver, 
ResourceDriver
 from . import (
 utils,
 parser,
@@ -58,37 +57,37 @@ def install_aria_extensions():
 del sys.modules[module_name]
 
 
-def application_model_storage(driver):
+def application_model_storage(api, api_params=None):
 """
 Initiate model storage for the supplied storage driver
 """
-
-assert isinstance(driver, ModelDriver)
-if driver not in _model_storage:
-_model_storage[driver] = ModelStorage(
-driver, model_classes=[
-models.Node,
-models.NodeInstance,
-models.Plugin,
-models.Blueprint,
-models.Snapshot,
-models.Deployment,
- 

[1/3] incubator-ariatosca git commit: ARIA-30 SQL based storage implementation [Forced Update!]

[mxmrlv]

Repository: incubator-ariatosca
Updated Branches:
  refs/heads/ARIA-30-SQL-based-storage-implementation f6981ddbf -> c9df5b140 
(forced update)


http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/c9df5b14/tests/orchestrator/workflows/api/test_task.py
--
diff --git a/tests/orchestrator/workflows/api/test_task.py 
b/tests/orchestrator/workflows/api/test_task.py
index 8536902..4da42c1 100644
--- a/tests/orchestrator/workflows/api/test_task.py
+++ b/tests/orchestrator/workflows/api/test_task.py
@@ -22,7 +22,7 @@ from aria.orchestrator.workflows import api
 from tests import mock
 
 
-@pytest.fixture()
+@pytest.fixture
 def ctx():
 """
 Create the following graph in storage:
@@ -30,50 +30,26 @@ def ctx():
 :return:
 """
 simple_context = mock.context.simple()
-dependency_node = mock.models.get_dependency_node()
-dependency_node_instance = mock.models.get_dependency_node_instance(
-dependency_node=dependency_node)
-
-relationship = mock.models.get_relationship(dependency_node)
-relationship_instance = mock.models.get_relationship_instance(
-relationship=relationship,
-target_instance=dependency_node_instance
-)
-
-dependent_node = mock.models.get_dependent_node(relationship)
-dependent_node_instance = mock.models.get_dependent_node_instance(
-dependent_node=dependent_node,
-relationship_instance=relationship_instance
-)
-
-simple_context.model.node.store(dependent_node)
-simple_context.model.node.store(dependency_node)
-simple_context.model.node_instance.store(dependent_node_instance)
-simple_context.model.node_instance.store(dependency_node_instance)
-simple_context.model.relationship.store(relationship)
-simple_context.model.relationship_instance.store(relationship_instance)
-simple_context.model.execution.store(mock.models.get_execution())
-simple_context.model.deployment.store(mock.models.get_deployment())
+
simple_context.model.execution.store(mock.models.get_execution(simple_context.deployment))
 
 return simple_context
 
 
 class TestOperationTask(object):
 
-def test_node_operation_task_creation(self):
-workflow_context = mock.context.simple()
-
+def test_node_operation_task_creation(self, ctx):
 operation_name = 'aria.interfaces.lifecycle.create'
 op_details = {'operation': True}
-node = mock.models.get_dependency_node()
+node = ctx.model.node.get(mock.models.DEPENDENT_NODE_ID)
 node.operations[operation_name] = op_details
-node_instance = 
mock.models.get_dependency_node_instance(dependency_node=node)
+ctx.model.node.update(node)
+node_instance = 
ctx.model.node_instance.get(mock.models.DEPENDENT_NODE_INSTANCE_ID)
 inputs = {'inputs': True}
 max_attempts = 10
 retry_interval = 10
 ignore_failure = True
 
-with context.workflow.current.push(workflow_context):
+with context.workflow.current.push(ctx):
 api_task = api.task.OperationTask.node_instance(
 name=operation_name,
 instance=node_instance,
@@ -90,19 +66,18 @@ class TestOperationTask(object):
 assert api_task.max_attempts == max_attempts
 assert api_task.ignore_failure == ignore_failure
 
-def test_relationship_operation_task_creation(self):
-workflow_context = mock.context.simple()
-
+def test_relationship_operation_task_creation(self, ctx):
 operation_name = 'aria.interfaces.relationship_lifecycle.preconfigure'
 op_details = {'operation': True}
-relationship = mock.models.get_relationship()
+relationship = ctx.model.relationship.get(mock.models.RELATIONSHIP_ID)
 relationship.source_operations[operation_name] = op_details
-relationship_instance = 
mock.models.get_relationship_instance(relationship=relationship)
+relationship_instance = ctx.model.relationship_instance.get(
+mock.models.RELATIONSHIP_INSTANCE_ID)
 inputs = {'inputs': True}
 max_attempts = 10
 retry_interval = 10
 
-with context.workflow.current.push(workflow_context):
+with context.workflow.current.push(ctx):
 api_task = api.task.OperationTask.relationship_instance(
 name=operation_name,
 instance=relationship_instance,
@@ -118,18 +93,19 @@ class TestOperationTask(object):
 assert api_task.retry_interval == retry_interval
 assert api_task.max_attempts == max_attempts
 
-def test_operation_task_default_values(self):
-workflow_context = mock.context.simple(task_ignore_failure=True)
-with context.workflow.current.push(workflow_context):
-model_task = api.task.OperationTask(
+def test_operation_task_default_values(self, ctx):
+dependency_node_instance = 

[3/3] incubator-ariatosca git commit: ARIA-30 SQL based storage implementation

[mxmrlv]

ARIA-30 SQL based storage implementation


Project: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/commit/f6981ddb
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/tree/f6981ddb
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/diff/f6981ddb

Branch: refs/heads/ARIA-30-SQL-based-storage-implementation
Commit: f6981ddbf31fd1adba12e65c725e77aba84b7c03
Parents: fe974e4
Author: mxmrlv 
Authored: Sun Nov 27 13:20:46 2016 +0200
Committer: mxmrlv 
Committed: Thu Dec 1 15:37:43 2016 +0200

--
 aria/__init__.py|  43 +-
 aria/orchestrator/__init__.py   |   4 +-
 aria/orchestrator/context/common.py |   2 +-
 aria/orchestrator/context/exceptions.py |   4 +-
 aria/orchestrator/context/operation.py  |   8 +-
 aria/orchestrator/context/toolbelt.py   |  13 +-
 aria/orchestrator/context/workflow.py   |  20 +-
 aria/orchestrator/exceptions.py |   7 +-
 aria/orchestrator/workflows/api/task.py |  10 +-
 aria/orchestrator/workflows/builtin/heal.py |  25 +-
 aria/orchestrator/workflows/builtin/install.py  |   7 +-
 .../orchestrator/workflows/builtin/uninstall.py |   5 +-
 .../orchestrator/workflows/builtin/workflows.py |   4 +-
 aria/orchestrator/workflows/core/task.py|  21 +-
 aria/storage/__init__.py| 379 ++--
 aria/storage/api.py | 219 +
 aria/storage/drivers.py | 416 -
 aria/storage/exceptions.py  |   4 +-
 aria/storage/filesystem_api.py  |  39 +
 aria/storage/mapi/__init__.py   |  20 +
 aria/storage/mapi/filesystem.py | 118 +++
 aria/storage/mapi/inmemory.py   | 148 +++
 aria/storage/mapi/sql.py| 368 
 aria/storage/models.py  | 912 +--
 aria/storage/rapi/__init__.py   |  18 +
 aria/storage/rapi/filesystem.py | 119 +++
 aria/storage/structures.py  | 402 
 requirements.txt|   1 +
 tests/mock/context.py   |  50 +-
 tests/mock/models.py|  68 +-
 tests/orchestrator/context/test_operation.py|  36 +-
 tests/orchestrator/context/test_toolbelt.py |  47 +-
 tests/orchestrator/context/test_workflow.py |  10 +-
 tests/orchestrator/workflows/api/test_task.py   |  68 +-
 .../orchestrator/workflows/builtin/__init__.py  |  35 +-
 .../workflows/builtin/test_execute_operation.py |  11 +-
 .../orchestrator/workflows/builtin/test_heal.py |  18 +-
 .../workflows/builtin/test_install.py   |  14 +-
 .../workflows/builtin/test_uninstall.py |  12 +-
 .../orchestrator/workflows/core/test_engine.py  |  71 +-
 tests/orchestrator/workflows/core/test_task.py  |  20 +-
 .../test_task_graph_into_exececution_graph.py   |  10 +-
 tests/requirements.txt  |   2 +-
 tests/storage/__init__.py   |  38 +-
 tests/storage/test_drivers.py   | 135 ---
 tests/storage/test_field.py | 124 ---
 tests/storage/test_model_storage.py | 167 ++--
 tests/storage/test_models.py| 364 
 tests/storage/test_models_api.py|  70 --
 tests/storage/test_resource_storage.py  |  57 +-
 50 files changed, 2295 insertions(+), 2468 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/f6981ddb/aria/__init__.py
--
diff --git a/aria/__init__.py b/aria/__init__.py
index 3f81f98..6e810f0 100644
--- a/aria/__init__.py
+++ b/aria/__init__.py
@@ -23,7 +23,6 @@ import pkgutil
 from .VERSION import version as __version__
 
 from .orchestrator.decorators import workflow, operation
-from .storage import ModelStorage, ResourceStorage, models, ModelDriver, 
ResourceDriver
 from . import (
 utils,
 parser,
@@ -58,37 +57,37 @@ def install_aria_extensions():
 del sys.modules[module_name]
 
 
-def application_model_storage(driver):
+def application_model_storage(api, api_params=None):
 """
 Initiate model storage for the supplied storage driver
 """
-
-assert isinstance(driver, ModelDriver)
-if driver not in _model_storage:
-_model_storage[driver] = ModelStorage(
-driver, model_classes=[
-models.Node,
-models.NodeInstance,
-models.Plugin,
-models.Blueprint,
-models.Snapshot,
-models.Deployment,
- 

[1/3] incubator-ariatosca git commit: ARIA-30 SQL based storage implementation [Forced Update!]

[mxmrlv]

Repository: incubator-ariatosca
Updated Branches:
  refs/heads/ARIA-30-SQL-based-storage-implementation 358cbe053 -> f6981ddbf 
(forced update)


http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/f6981ddb/tests/orchestrator/workflows/api/test_task.py
--
diff --git a/tests/orchestrator/workflows/api/test_task.py 
b/tests/orchestrator/workflows/api/test_task.py
index 8536902..4da42c1 100644
--- a/tests/orchestrator/workflows/api/test_task.py
+++ b/tests/orchestrator/workflows/api/test_task.py
@@ -22,7 +22,7 @@ from aria.orchestrator.workflows import api
 from tests import mock
 
 
-@pytest.fixture()
+@pytest.fixture
 def ctx():
 """
 Create the following graph in storage:
@@ -30,50 +30,26 @@ def ctx():
 :return:
 """
 simple_context = mock.context.simple()
-dependency_node = mock.models.get_dependency_node()
-dependency_node_instance = mock.models.get_dependency_node_instance(
-dependency_node=dependency_node)
-
-relationship = mock.models.get_relationship(dependency_node)
-relationship_instance = mock.models.get_relationship_instance(
-relationship=relationship,
-target_instance=dependency_node_instance
-)
-
-dependent_node = mock.models.get_dependent_node(relationship)
-dependent_node_instance = mock.models.get_dependent_node_instance(
-dependent_node=dependent_node,
-relationship_instance=relationship_instance
-)
-
-simple_context.model.node.store(dependent_node)
-simple_context.model.node.store(dependency_node)
-simple_context.model.node_instance.store(dependent_node_instance)
-simple_context.model.node_instance.store(dependency_node_instance)
-simple_context.model.relationship.store(relationship)
-simple_context.model.relationship_instance.store(relationship_instance)
-simple_context.model.execution.store(mock.models.get_execution())
-simple_context.model.deployment.store(mock.models.get_deployment())
+
simple_context.model.execution.store(mock.models.get_execution(simple_context.deployment))
 
 return simple_context
 
 
 class TestOperationTask(object):
 
-def test_node_operation_task_creation(self):
-workflow_context = mock.context.simple()
-
+def test_node_operation_task_creation(self, ctx):
 operation_name = 'aria.interfaces.lifecycle.create'
 op_details = {'operation': True}
-node = mock.models.get_dependency_node()
+node = ctx.model.node.get(mock.models.DEPENDENT_NODE_ID)
 node.operations[operation_name] = op_details
-node_instance = 
mock.models.get_dependency_node_instance(dependency_node=node)
+ctx.model.node.update(node)
+node_instance = 
ctx.model.node_instance.get(mock.models.DEPENDENT_NODE_INSTANCE_ID)
 inputs = {'inputs': True}
 max_attempts = 10
 retry_interval = 10
 ignore_failure = True
 
-with context.workflow.current.push(workflow_context):
+with context.workflow.current.push(ctx):
 api_task = api.task.OperationTask.node_instance(
 name=operation_name,
 instance=node_instance,
@@ -90,19 +66,18 @@ class TestOperationTask(object):
 assert api_task.max_attempts == max_attempts
 assert api_task.ignore_failure == ignore_failure
 
-def test_relationship_operation_task_creation(self):
-workflow_context = mock.context.simple()
-
+def test_relationship_operation_task_creation(self, ctx):
 operation_name = 'aria.interfaces.relationship_lifecycle.preconfigure'
 op_details = {'operation': True}
-relationship = mock.models.get_relationship()
+relationship = ctx.model.relationship.get(mock.models.RELATIONSHIP_ID)
 relationship.source_operations[operation_name] = op_details
-relationship_instance = 
mock.models.get_relationship_instance(relationship=relationship)
+relationship_instance = ctx.model.relationship_instance.get(
+mock.models.RELATIONSHIP_INSTANCE_ID)
 inputs = {'inputs': True}
 max_attempts = 10
 retry_interval = 10
 
-with context.workflow.current.push(workflow_context):
+with context.workflow.current.push(ctx):
 api_task = api.task.OperationTask.relationship_instance(
 name=operation_name,
 instance=relationship_instance,
@@ -118,18 +93,19 @@ class TestOperationTask(object):
 assert api_task.retry_interval == retry_interval
 assert api_task.max_attempts == max_attempts
 
-def test_operation_task_default_values(self):
-workflow_context = mock.context.simple(task_ignore_failure=True)
-with context.workflow.current.push(workflow_context):
-model_task = api.task.OperationTask(
+def test_operation_task_default_values(self, ctx):
+dependency_node_instance = 

[2/3] incubator-ariatosca git commit: ARIA-30 SQL based storage implementation

[mxmrlv]

http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/f6981ddb/aria/storage/mapi/sql.py
--
diff --git a/aria/storage/mapi/sql.py b/aria/storage/mapi/sql.py
new file mode 100644
index 000..652dc9f
--- /dev/null
+++ b/aria/storage/mapi/sql.py
@@ -0,0 +1,368 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+SQLAlchemy based MAPI
+"""
+
+from sqlite3 import DatabaseError as SQLiteDBError
+from sqlalchemy.exc import SQLAlchemyError
+from sqlalchemy.sql.elements import Label
+
+try:
+from psycopg2 import DatabaseError as Psycopg2DBError
+sql_errors = (SQLAlchemyError, SQLiteDBError, Psycopg2DBError)
+except ImportError:
+sql_errors = (SQLAlchemyError, SQLiteDBError)
+Psycopg2DBError = None
+
+from aria.utils.collections import OrderedDict
+
+from ... import storage
+
+
+DEFAULT_SQL_DIALECT = 'sqlite'
+
+
+class SQLAlchemyModelAPI(storage.api.ModelAPI):
+"""
+SQL based MAPI.
+"""
+
+def __init__(self,
+ engine,
+ session,
+ **kwargs):
+super(SQLAlchemyModelAPI, self).__init__(**kwargs)
+self._engine = engine
+self._session = session
+
+def get(self, entry_id, include=None, filters=None, locking=False, 
**kwargs):
+"""Return a single result based on the model class and element ID
+"""
+filters = filters or {'id': entry_id}
+query = self._get_query(include, filters)
+if locking:
+query = query.with_for_update()
+result = query.first()
+
+if not result:
+raise storage.exceptions.StorageError(
+'Requested {0} with ID `{1}` was not found'
+.format(self.model_cls.__name__, entry_id)
+)
+return result
+
+def iter(self,
+ include=None,
+ filters=None,
+ pagination=None,
+ sort=None,
+ **kwargs):
+"""Return a (possibly empty) list of `model_class` results
+"""
+query = self._get_query(include, filters, sort)
+
+results, _, _, _ = self._paginate(query, pagination)
+
+for result in results:
+yield result
+
+def store(self, entry, **kwargs):
+"""Create a `model_class` instance from a serializable `model` object
+
+:param entry: A dict with relevant kwargs, or an instance of a class
+that has a `to_dict` method, and whose attributes match the columns
+of `model_class` (might also my just an instance of `model_class`)
+:return: An instance of `model_class`
+"""
+self._session.add(entry)
+self._safe_commit()
+return entry
+
+def delete(self, entry_id, filters=None, **kwargs):
+"""Delete a single result based on the model class and element ID
+"""
+try:
+instance = self.get(
+entry_id,
+filters=filters
+)
+except storage.exceptions.StorageError:
+raise storage.exceptions.StorageError(
+'Could not delete {0} with ID `{1}` - element not found'
+.format(
+self.model_cls.__name__,
+entry_id
+)
+)
+self._load_properties(instance)
+self._session.delete(instance)
+self._safe_commit()
+return instance
+
+# TODO: this might need rework
+def update(self, entry, **kwargs):
+"""Add `instance` to the DB session, and attempt to commit
+
+:return: The updated instance
+"""
+return self.store(entry)
+
+def refresh(self, entry):
+"""Reload the instance with fresh information from the DB
+
+:param entry: Instance to be re-loaded from the DB
+:return: The refreshed instance
+"""
+self._session.refresh(entry)
+self._load_properties(entry)
+return entry
+
+def _destroy_connection(self):
+pass
+
+def _establish_connection(self):
+pass
+
+def create(self):
+self.model_cls.__table__.create(self._engine)
+
+def drop(self):
+"""
+

[2/3] incubator-ariatosca git commit: ARIA-30 SQL based storage implementation

[mxmrlv]

http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/358cbe05/aria/storage/mapi/sql.py
--
diff --git a/aria/storage/mapi/sql.py b/aria/storage/mapi/sql.py
new file mode 100644
index 000..4408aa3
--- /dev/null
+++ b/aria/storage/mapi/sql.py
@@ -0,0 +1,369 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+SQLalchemy based MAPI
+"""
+
+from sqlite3 import DatabaseError as SQLiteDBError
+from sqlalchemy.exc import SQLAlchemyError
+from sqlalchemy.sql.elements import Label
+
+from aria.utils.collections import OrderedDict
+
+
+try:
+from psycopg2 import DatabaseError as Psycopg2DBError
+sql_errors = (SQLAlchemyError, SQLiteDBError, Psycopg2DBError)
+except ImportError:
+sql_errors = (SQLAlchemyError, SQLiteDBError)
+Psycopg2DBError = None
+
+from ... import storage
+
+
+DEFAULT_SQL_DIALECT = 'sqlite'
+
+
+class SQLAlchemyModelAPI(storage.api.ModelAPI):
+"""
+SQL based MAPI.
+"""
+
+def __init__(self,
+ engine,
+ session,
+ **kwargs):
+super(SQLAlchemyModelAPI, self).__init__(**kwargs)
+self._engine = engine
+self._session = session
+
+def get(self, entry_id, include=None, filters=None, locking=False, 
**kwargs):
+"""Return a single result based on the model class and element ID
+"""
+filters = filters or {'id': entry_id}
+query = self._get_query(include, filters)
+if locking:
+query = query.with_for_update()
+result = query.first()
+
+if not result:
+raise storage.exceptions.StorageError(
+'Requested {0} with ID `{1}` was not found'
+.format(self.model_cls.__name__, entry_id)
+)
+return result
+
+def iter(self,
+ include=None,
+ filters=None,
+ pagination=None,
+ sort=None,
+ **kwargs):
+"""Return a (possibly empty) list of `model_class` results
+"""
+query = self._get_query(include, filters, sort)
+
+results, _, _, _ = self._paginate(query, pagination)
+
+for result in results:
+yield result
+
+def store(self, entry, **kwargs):
+"""Create a `model_class` instance from a serializable `model` object
+
+:param entry: A dict with relevant kwargs, or an instance of a class
+that has a `to_dict` method, and whose attributes match the columns
+of `model_class` (might also my just an instance of `model_class`)
+:return: An instance of `model_class`
+"""
+self._session.add(entry)
+self._safe_commit()
+return entry
+
+def delete(self, entry_id, filters=None, **kwargs):
+"""Delete a single result based on the model class and element ID
+"""
+try:
+instance = self.get(
+entry_id,
+filters=filters
+)
+except storage.exceptions.StorageError:
+raise storage.exceptions.StorageError(
+'Could not delete {0} with ID `{1}` - element not found'
+.format(
+self.model_cls.__name__,
+entry_id
+)
+)
+self._load_properties(instance)
+self._session.delete(instance)
+self._safe_commit()
+return instance
+
+# TODO: this might need rework
+def update(self, entry, **kwargs):
+"""Add `instance` to the DB session, and attempt to commit
+
+:return: The updated instance
+"""
+return self.store(entry)
+
+def refresh(self, entry):
+"""Reload the instance with fresh information from the DB
+
+:param entry: Instance to be re-loaded from the DB
+:return: The refreshed instance
+"""
+self._session.refresh(entry)
+self._load_properties(entry)
+return entry
+
+def _destroy_connection(self):
+pass
+
+def _establish_connection(self):
+pass
+
+def create(self):
+self.model_cls.__table__.create(self._engine)
+
+def drop(self):
+"""
+  

[1/3] incubator-ariatosca git commit: ARIA-30 SQL based storage implementation [Forced Update!]

[mxmrlv]

Repository: incubator-ariatosca
Updated Branches:
  refs/heads/ARIA-30-SQL-based-storage-implementation fe944e58d -> 358cbe053 
(forced update)


http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/358cbe05/tests/orchestrator/workflows/api/test_task.py
--
diff --git a/tests/orchestrator/workflows/api/test_task.py 
b/tests/orchestrator/workflows/api/test_task.py
index 8536902..4da42c1 100644
--- a/tests/orchestrator/workflows/api/test_task.py
+++ b/tests/orchestrator/workflows/api/test_task.py
@@ -22,7 +22,7 @@ from aria.orchestrator.workflows import api
 from tests import mock
 
 
-@pytest.fixture()
+@pytest.fixture
 def ctx():
 """
 Create the following graph in storage:
@@ -30,50 +30,26 @@ def ctx():
 :return:
 """
 simple_context = mock.context.simple()
-dependency_node = mock.models.get_dependency_node()
-dependency_node_instance = mock.models.get_dependency_node_instance(
-dependency_node=dependency_node)
-
-relationship = mock.models.get_relationship(dependency_node)
-relationship_instance = mock.models.get_relationship_instance(
-relationship=relationship,
-target_instance=dependency_node_instance
-)
-
-dependent_node = mock.models.get_dependent_node(relationship)
-dependent_node_instance = mock.models.get_dependent_node_instance(
-dependent_node=dependent_node,
-relationship_instance=relationship_instance
-)
-
-simple_context.model.node.store(dependent_node)
-simple_context.model.node.store(dependency_node)
-simple_context.model.node_instance.store(dependent_node_instance)
-simple_context.model.node_instance.store(dependency_node_instance)
-simple_context.model.relationship.store(relationship)
-simple_context.model.relationship_instance.store(relationship_instance)
-simple_context.model.execution.store(mock.models.get_execution())
-simple_context.model.deployment.store(mock.models.get_deployment())
+
simple_context.model.execution.store(mock.models.get_execution(simple_context.deployment))
 
 return simple_context
 
 
 class TestOperationTask(object):
 
-def test_node_operation_task_creation(self):
-workflow_context = mock.context.simple()
-
+def test_node_operation_task_creation(self, ctx):
 operation_name = 'aria.interfaces.lifecycle.create'
 op_details = {'operation': True}
-node = mock.models.get_dependency_node()
+node = ctx.model.node.get(mock.models.DEPENDENT_NODE_ID)
 node.operations[operation_name] = op_details
-node_instance = 
mock.models.get_dependency_node_instance(dependency_node=node)
+ctx.model.node.update(node)
+node_instance = 
ctx.model.node_instance.get(mock.models.DEPENDENT_NODE_INSTANCE_ID)
 inputs = {'inputs': True}
 max_attempts = 10
 retry_interval = 10
 ignore_failure = True
 
-with context.workflow.current.push(workflow_context):
+with context.workflow.current.push(ctx):
 api_task = api.task.OperationTask.node_instance(
 name=operation_name,
 instance=node_instance,
@@ -90,19 +66,18 @@ class TestOperationTask(object):
 assert api_task.max_attempts == max_attempts
 assert api_task.ignore_failure == ignore_failure
 
-def test_relationship_operation_task_creation(self):
-workflow_context = mock.context.simple()
-
+def test_relationship_operation_task_creation(self, ctx):
 operation_name = 'aria.interfaces.relationship_lifecycle.preconfigure'
 op_details = {'operation': True}
-relationship = mock.models.get_relationship()
+relationship = ctx.model.relationship.get(mock.models.RELATIONSHIP_ID)
 relationship.source_operations[operation_name] = op_details
-relationship_instance = 
mock.models.get_relationship_instance(relationship=relationship)
+relationship_instance = ctx.model.relationship_instance.get(
+mock.models.RELATIONSHIP_INSTANCE_ID)
 inputs = {'inputs': True}
 max_attempts = 10
 retry_interval = 10
 
-with context.workflow.current.push(workflow_context):
+with context.workflow.current.push(ctx):
 api_task = api.task.OperationTask.relationship_instance(
 name=operation_name,
 instance=relationship_instance,
@@ -118,18 +93,19 @@ class TestOperationTask(object):
 assert api_task.retry_interval == retry_interval
 assert api_task.max_attempts == max_attempts
 
-def test_operation_task_default_values(self):
-workflow_context = mock.context.simple(task_ignore_failure=True)
-with context.workflow.current.push(workflow_context):
-model_task = api.task.OperationTask(
+def test_operation_task_default_values(self, ctx):
+dependency_node_instance = 

[3/3] incubator-ariatosca git commit: ARIA-30 SQL based storage implementation

[mxmrlv]

ARIA-30 SQL based storage implementation


Project: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/commit/358cbe05
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/tree/358cbe05
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/diff/358cbe05

Branch: refs/heads/ARIA-30-SQL-based-storage-implementation
Commit: 358cbe053190c20d0674252345586bd8b199809f
Parents: fe974e4
Author: mxmrlv 
Authored: Sun Nov 27 13:20:46 2016 +0200
Committer: mxmrlv 
Committed: Thu Dec 1 14:49:48 2016 +0200

--
 aria/__init__.py|  43 +-
 aria/orchestrator/__init__.py   |   4 +-
 aria/orchestrator/context/common.py |   2 +-
 aria/orchestrator/context/exceptions.py |   4 +-
 aria/orchestrator/context/operation.py  |   8 +-
 aria/orchestrator/context/toolbelt.py   |  13 +-
 aria/orchestrator/context/workflow.py   |  20 +-
 aria/orchestrator/exceptions.py |   7 +-
 aria/orchestrator/workflows/api/task.py |  10 +-
 aria/orchestrator/workflows/builtin/heal.py |  25 +-
 aria/orchestrator/workflows/builtin/install.py  |   7 +-
 .../orchestrator/workflows/builtin/uninstall.py |   5 +-
 .../orchestrator/workflows/builtin/workflows.py |   4 +-
 aria/orchestrator/workflows/core/task.py|  21 +-
 aria/storage/__init__.py| 379 ++--
 aria/storage/api.py | 219 +
 aria/storage/drivers.py | 416 -
 aria/storage/exceptions.py  |   4 +-
 aria/storage/filesystem_api.py  |  39 +
 aria/storage/mapi/__init__.py   |  20 +
 aria/storage/mapi/filesystem.py | 118 +++
 aria/storage/mapi/inmemory.py   | 148 +++
 aria/storage/mapi/sql.py| 369 
 aria/storage/models.py  | 912 +--
 aria/storage/rapi/__init__.py   |  18 +
 aria/storage/rapi/filesystem.py | 119 +++
 aria/storage/structures.py  | 402 
 requirements.txt|   1 +
 tests/mock/context.py   |  50 +-
 tests/mock/models.py|  68 +-
 tests/orchestrator/context/test_operation.py|  36 +-
 tests/orchestrator/context/test_toolbelt.py |  47 +-
 tests/orchestrator/context/test_workflow.py |  10 +-
 tests/orchestrator/workflows/api/test_task.py   |  68 +-
 .../orchestrator/workflows/builtin/__init__.py  |  35 +-
 .../workflows/builtin/test_execute_operation.py |  11 +-
 .../orchestrator/workflows/builtin/test_heal.py |  18 +-
 .../workflows/builtin/test_install.py   |  14 +-
 .../workflows/builtin/test_uninstall.py |  12 +-
 .../orchestrator/workflows/core/test_engine.py  |  71 +-
 tests/orchestrator/workflows/core/test_task.py  |  20 +-
 .../test_task_graph_into_exececution_graph.py   |  10 +-
 tests/requirements.txt  |   2 +-
 tests/storage/__init__.py   |  38 +-
 tests/storage/test_drivers.py   | 135 ---
 tests/storage/test_field.py | 124 ---
 tests/storage/test_model_storage.py | 167 ++--
 tests/storage/test_models.py| 364 
 tests/storage/test_models_api.py|  70 --
 tests/storage/test_resource_storage.py  |  57 +-
 50 files changed, 2296 insertions(+), 2468 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/358cbe05/aria/__init__.py
--
diff --git a/aria/__init__.py b/aria/__init__.py
index 3f81f98..6e810f0 100644
--- a/aria/__init__.py
+++ b/aria/__init__.py
@@ -23,7 +23,6 @@ import pkgutil
 from .VERSION import version as __version__
 
 from .orchestrator.decorators import workflow, operation
-from .storage import ModelStorage, ResourceStorage, models, ModelDriver, 
ResourceDriver
 from . import (
 utils,
 parser,
@@ -58,37 +57,37 @@ def install_aria_extensions():
 del sys.modules[module_name]
 
 
-def application_model_storage(driver):
+def application_model_storage(api, api_params=None):
 """
 Initiate model storage for the supplied storage driver
 """
-
-assert isinstance(driver, ModelDriver)
-if driver not in _model_storage:
-_model_storage[driver] = ModelStorage(
-driver, model_classes=[
-models.Node,
-models.NodeInstance,
-models.Plugin,
-models.Blueprint,
-models.Snapshot,
-models.Deployment,
- 

[3/3] incubator-ariatosca git commit: ARIA-30-SQL-based-storage-implementation

[mxmrlv]

ARIA-30-SQL-based-storage-implementation


Project: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/commit/f68e11bf
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/tree/f68e11bf
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/diff/f68e11bf

Branch: refs/heads/ARIA-30-SQL-based-storage-implementation
Commit: f68e11bfb8ea6e59a496887773af321a46bcd81d
Parents: fe974e4
Author: mxmrlv 
Authored: Sun Nov 27 13:20:46 2016 +0200
Committer: mxmrlv 
Committed: Thu Dec 1 14:36:13 2016 +0200

--
 aria/__init__.py|  43 +-
 aria/orchestrator/__init__.py   |   4 +-
 aria/orchestrator/context/common.py |   2 +-
 aria/orchestrator/context/exceptions.py |   4 +-
 aria/orchestrator/context/operation.py  |   8 +-
 aria/orchestrator/context/toolbelt.py   |  13 +-
 aria/orchestrator/context/workflow.py   |  20 +-
 aria/orchestrator/exceptions.py |   7 +-
 aria/orchestrator/workflows/api/task.py |  10 +-
 aria/orchestrator/workflows/builtin/heal.py |  25 +-
 aria/orchestrator/workflows/builtin/install.py  |   7 +-
 .../orchestrator/workflows/builtin/uninstall.py |   5 +-
 .../orchestrator/workflows/builtin/workflows.py |   4 +-
 aria/orchestrator/workflows/core/task.py|  21 +-
 aria/storage/__init__.py| 379 ++--
 aria/storage/api.py | 219 +
 aria/storage/drivers.py | 416 -
 aria/storage/exceptions.py  |   4 +-
 aria/storage/filesystem_api.py  |  39 +
 aria/storage/mapi/__init__.py   |  20 +
 aria/storage/mapi/filesystem.py | 118 +++
 aria/storage/mapi/inmemory.py   | 148 +++
 aria/storage/mapi/sql.py| 369 
 aria/storage/models.py  | 912 +--
 aria/storage/rapi/__init__.py   |  18 +
 aria/storage/rapi/filesystem.py | 119 +++
 aria/storage/structures.py  | 424 -
 requirements.txt|   1 +
 tests/mock/context.py   |  50 +-
 tests/mock/models.py|  68 +-
 tests/orchestrator/context/test_operation.py|  36 +-
 tests/orchestrator/context/test_toolbelt.py |  47 +-
 tests/orchestrator/context/test_workflow.py |  10 +-
 tests/orchestrator/workflows/api/test_task.py   |  68 +-
 .../orchestrator/workflows/builtin/__init__.py  |  35 +-
 .../workflows/builtin/test_execute_operation.py |  11 +-
 .../orchestrator/workflows/builtin/test_heal.py |  18 +-
 .../workflows/builtin/test_install.py   |  14 +-
 .../workflows/builtin/test_uninstall.py |  12 +-
 .../orchestrator/workflows/core/test_engine.py  |  71 +-
 tests/orchestrator/workflows/core/test_task.py  |  20 +-
 .../test_task_graph_into_exececution_graph.py   |  10 +-
 tests/requirements.txt  |   2 +-
 tests/storage/__init__.py   |  38 +-
 tests/storage/test_drivers.py   | 135 ---
 tests/storage/test_field.py | 124 ---
 tests/storage/test_model_storage.py | 167 ++--
 tests/storage/test_models.py| 364 
 tests/storage/test_models_api.py|  70 --
 tests/storage/test_resource_storage.py  |  57 +-
 50 files changed, 2318 insertions(+), 2468 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/f68e11bf/aria/__init__.py
--
diff --git a/aria/__init__.py b/aria/__init__.py
index 3f81f98..6e810f0 100644
--- a/aria/__init__.py
+++ b/aria/__init__.py
@@ -23,7 +23,6 @@ import pkgutil
 from .VERSION import version as __version__
 
 from .orchestrator.decorators import workflow, operation
-from .storage import ModelStorage, ResourceStorage, models, ModelDriver, 
ResourceDriver
 from . import (
 utils,
 parser,
@@ -58,37 +57,37 @@ def install_aria_extensions():
 del sys.modules[module_name]
 
 
-def application_model_storage(driver):
+def application_model_storage(api, api_params=None):
 """
 Initiate model storage for the supplied storage driver
 """
-
-assert isinstance(driver, ModelDriver)
-if driver not in _model_storage:
-_model_storage[driver] = ModelStorage(
-driver, model_classes=[
-models.Node,
-models.NodeInstance,
-models.Plugin,
-models.Blueprint,
-models.Snapshot,
-models.Deployment,
-

[2/3] incubator-ariatosca git commit: ARIA-30-SQL-based-storage-implementation

[mxmrlv]

http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/f68e11bf/aria/storage/mapi/sql.py
--
diff --git a/aria/storage/mapi/sql.py b/aria/storage/mapi/sql.py
new file mode 100644
index 000..4408aa3
--- /dev/null
+++ b/aria/storage/mapi/sql.py
@@ -0,0 +1,369 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+SQLalchemy based MAPI
+"""
+
+from sqlite3 import DatabaseError as SQLiteDBError
+from sqlalchemy.exc import SQLAlchemyError
+from sqlalchemy.sql.elements import Label
+
+from aria.utils.collections import OrderedDict
+
+
+try:
+from psycopg2 import DatabaseError as Psycopg2DBError
+sql_errors = (SQLAlchemyError, SQLiteDBError, Psycopg2DBError)
+except ImportError:
+sql_errors = (SQLAlchemyError, SQLiteDBError)
+Psycopg2DBError = None
+
+from ... import storage
+
+
+DEFAULT_SQL_DIALECT = 'sqlite'
+
+
+class SQLAlchemyModelAPI(storage.api.ModelAPI):
+"""
+SQL based MAPI.
+"""
+
+def __init__(self,
+ engine,
+ session,
+ **kwargs):
+super(SQLAlchemyModelAPI, self).__init__(**kwargs)
+self._engine = engine
+self._session = session
+
+def get(self, entry_id, include=None, filters=None, locking=False, 
**kwargs):
+"""Return a single result based on the model class and element ID
+"""
+filters = filters or {'id': entry_id}
+query = self._get_query(include, filters)
+if locking:
+query = query.with_for_update()
+result = query.first()
+
+if not result:
+raise storage.exceptions.StorageError(
+'Requested {0} with ID `{1}` was not found'
+.format(self.model_cls.__name__, entry_id)
+)
+return result
+
+def iter(self,
+ include=None,
+ filters=None,
+ pagination=None,
+ sort=None,
+ **kwargs):
+"""Return a (possibly empty) list of `model_class` results
+"""
+query = self._get_query(include, filters, sort)
+
+results, _, _, _ = self._paginate(query, pagination)
+
+for result in results:
+yield result
+
+def store(self, entry, **kwargs):
+"""Create a `model_class` instance from a serializable `model` object
+
+:param entry: A dict with relevant kwargs, or an instance of a class
+that has a `to_dict` method, and whose attributes match the columns
+of `model_class` (might also my just an instance of `model_class`)
+:return: An instance of `model_class`
+"""
+self._session.add(entry)
+self._safe_commit()
+return entry
+
+def delete(self, entry_id, filters=None, **kwargs):
+"""Delete a single result based on the model class and element ID
+"""
+try:
+instance = self.get(
+entry_id,
+filters=filters
+)
+except storage.exceptions.StorageError:
+raise storage.exceptions.StorageError(
+'Could not delete {0} with ID `{1}` - element not found'
+.format(
+self.model_cls.__name__,
+entry_id
+)
+)
+self._load_properties(instance)
+self._session.delete(instance)
+self._safe_commit()
+return instance
+
+# TODO: this might need rework
+def update(self, entry, **kwargs):
+"""Add `instance` to the DB session, and attempt to commit
+
+:return: The updated instance
+"""
+return self.store(entry)
+
+def refresh(self, entry):
+"""Reload the instance with fresh information from the DB
+
+:param entry: Instance to be re-loaded from the DB
+:return: The refreshed instance
+"""
+self._session.refresh(entry)
+self._load_properties(entry)
+return entry
+
+def _destroy_connection(self):
+pass
+
+def _establish_connection(self):
+pass
+
+def create(self):
+self.model_cls.__table__.create(self._engine)
+
+def drop(self):
+"""
+  

[4/6] incubator-ariatosca git commit: Storage is now sql based with SQLAlchemy based models

[mxmrlv]

http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/88bc5d18/tests/orchestrator/workflows/api/test_task.py
--
diff --git a/tests/orchestrator/workflows/api/test_task.py 
b/tests/orchestrator/workflows/api/test_task.py
index 8536902..4da42c1 100644
--- a/tests/orchestrator/workflows/api/test_task.py
+++ b/tests/orchestrator/workflows/api/test_task.py
@@ -22,7 +22,7 @@ from aria.orchestrator.workflows import api
 from tests import mock
 
 
-@pytest.fixture()
+@pytest.fixture
 def ctx():
 """
 Create the following graph in storage:
@@ -30,50 +30,26 @@ def ctx():
 :return:
 """
 simple_context = mock.context.simple()
-dependency_node = mock.models.get_dependency_node()
-dependency_node_instance = mock.models.get_dependency_node_instance(
-dependency_node=dependency_node)
-
-relationship = mock.models.get_relationship(dependency_node)
-relationship_instance = mock.models.get_relationship_instance(
-relationship=relationship,
-target_instance=dependency_node_instance
-)
-
-dependent_node = mock.models.get_dependent_node(relationship)
-dependent_node_instance = mock.models.get_dependent_node_instance(
-dependent_node=dependent_node,
-relationship_instance=relationship_instance
-)
-
-simple_context.model.node.store(dependent_node)
-simple_context.model.node.store(dependency_node)
-simple_context.model.node_instance.store(dependent_node_instance)
-simple_context.model.node_instance.store(dependency_node_instance)
-simple_context.model.relationship.store(relationship)
-simple_context.model.relationship_instance.store(relationship_instance)
-simple_context.model.execution.store(mock.models.get_execution())
-simple_context.model.deployment.store(mock.models.get_deployment())
+
simple_context.model.execution.store(mock.models.get_execution(simple_context.deployment))
 
 return simple_context
 
 
 class TestOperationTask(object):
 
-def test_node_operation_task_creation(self):
-workflow_context = mock.context.simple()
-
+def test_node_operation_task_creation(self, ctx):
 operation_name = 'aria.interfaces.lifecycle.create'
 op_details = {'operation': True}
-node = mock.models.get_dependency_node()
+node = ctx.model.node.get(mock.models.DEPENDENT_NODE_ID)
 node.operations[operation_name] = op_details
-node_instance = 
mock.models.get_dependency_node_instance(dependency_node=node)
+ctx.model.node.update(node)
+node_instance = 
ctx.model.node_instance.get(mock.models.DEPENDENT_NODE_INSTANCE_ID)
 inputs = {'inputs': True}
 max_attempts = 10
 retry_interval = 10
 ignore_failure = True
 
-with context.workflow.current.push(workflow_context):
+with context.workflow.current.push(ctx):
 api_task = api.task.OperationTask.node_instance(
 name=operation_name,
 instance=node_instance,
@@ -90,19 +66,18 @@ class TestOperationTask(object):
 assert api_task.max_attempts == max_attempts
 assert api_task.ignore_failure == ignore_failure
 
-def test_relationship_operation_task_creation(self):
-workflow_context = mock.context.simple()
-
+def test_relationship_operation_task_creation(self, ctx):
 operation_name = 'aria.interfaces.relationship_lifecycle.preconfigure'
 op_details = {'operation': True}
-relationship = mock.models.get_relationship()
+relationship = ctx.model.relationship.get(mock.models.RELATIONSHIP_ID)
 relationship.source_operations[operation_name] = op_details
-relationship_instance = 
mock.models.get_relationship_instance(relationship=relationship)
+relationship_instance = ctx.model.relationship_instance.get(
+mock.models.RELATIONSHIP_INSTANCE_ID)
 inputs = {'inputs': True}
 max_attempts = 10
 retry_interval = 10
 
-with context.workflow.current.push(workflow_context):
+with context.workflow.current.push(ctx):
 api_task = api.task.OperationTask.relationship_instance(
 name=operation_name,
 instance=relationship_instance,
@@ -118,18 +93,19 @@ class TestOperationTask(object):
 assert api_task.retry_interval == retry_interval
 assert api_task.max_attempts == max_attempts
 
-def test_operation_task_default_values(self):
-workflow_context = mock.context.simple(task_ignore_failure=True)
-with context.workflow.current.push(workflow_context):
-model_task = api.task.OperationTask(
+def test_operation_task_default_values(self, ctx):
+dependency_node_instance = ctx.model.node_instance.get(
+mock.models.DEPENDENCY_NODE_INSTANCE_ID)
+with context.workflow.current.push(ctx):
+task = 

[2/3] incubator-ariatosca git commit: ARIA-30 SQL based storage implementation

[mxmrlv]

http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/fe944e58/aria/storage/mapi/sql.py
--
diff --git a/aria/storage/mapi/sql.py b/aria/storage/mapi/sql.py
new file mode 100644
index 000..4408aa3
--- /dev/null
+++ b/aria/storage/mapi/sql.py
@@ -0,0 +1,369 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+SQLalchemy based MAPI
+"""
+
+from sqlite3 import DatabaseError as SQLiteDBError
+from sqlalchemy.exc import SQLAlchemyError
+from sqlalchemy.sql.elements import Label
+
+from aria.utils.collections import OrderedDict
+
+
+try:
+from psycopg2 import DatabaseError as Psycopg2DBError
+sql_errors = (SQLAlchemyError, SQLiteDBError, Psycopg2DBError)
+except ImportError:
+sql_errors = (SQLAlchemyError, SQLiteDBError)
+Psycopg2DBError = None
+
+from ... import storage
+
+
+DEFAULT_SQL_DIALECT = 'sqlite'
+
+
+class SQLAlchemyModelAPI(storage.api.ModelAPI):
+"""
+SQL based MAPI.
+"""
+
+def __init__(self,
+ engine,
+ session,
+ **kwargs):
+super(SQLAlchemyModelAPI, self).__init__(**kwargs)
+self._engine = engine
+self._session = session
+
+def get(self, entry_id, include=None, filters=None, locking=False, 
**kwargs):
+"""Return a single result based on the model class and element ID
+"""
+filters = filters or {'id': entry_id}
+query = self._get_query(include, filters)
+if locking:
+query = query.with_for_update()
+result = query.first()
+
+if not result:
+raise storage.exceptions.StorageError(
+'Requested {0} with ID `{1}` was not found'
+.format(self.model_cls.__name__, entry_id)
+)
+return result
+
+def iter(self,
+ include=None,
+ filters=None,
+ pagination=None,
+ sort=None,
+ **kwargs):
+"""Return a (possibly empty) list of `model_class` results
+"""
+query = self._get_query(include, filters, sort)
+
+results, _, _, _ = self._paginate(query, pagination)
+
+for result in results:
+yield result
+
+def store(self, entry, **kwargs):
+"""Create a `model_class` instance from a serializable `model` object
+
+:param entry: A dict with relevant kwargs, or an instance of a class
+that has a `to_dict` method, and whose attributes match the columns
+of `model_class` (might also my just an instance of `model_class`)
+:return: An instance of `model_class`
+"""
+self._session.add(entry)
+self._safe_commit()
+return entry
+
+def delete(self, entry_id, filters=None, **kwargs):
+"""Delete a single result based on the model class and element ID
+"""
+try:
+instance = self.get(
+entry_id,
+filters=filters
+)
+except storage.exceptions.StorageError:
+raise storage.exceptions.StorageError(
+'Could not delete {0} with ID `{1}` - element not found'
+.format(
+self.model_cls.__name__,
+entry_id
+)
+)
+self._load_properties(instance)
+self._session.delete(instance)
+self._safe_commit()
+return instance
+
+# TODO: this might need rework
+def update(self, entry, **kwargs):
+"""Add `instance` to the DB session, and attempt to commit
+
+:return: The updated instance
+"""
+return self.store(entry)
+
+def refresh(self, entry):
+"""Reload the instance with fresh information from the DB
+
+:param entry: Instance to be re-loaded from the DB
+:return: The refreshed instance
+"""
+self._session.refresh(entry)
+self._load_properties(entry)
+return entry
+
+def _destroy_connection(self):
+pass
+
+def _establish_connection(self):
+pass
+
+def create(self):
+self.model_cls.__table__.create(self._engine)
+
+def drop(self):
+"""
+  

[1/3] incubator-ariatosca git commit: ARIA-30 SQL based storage implementation [Forced Update!]

[mxmrlv]

Repository: incubator-ariatosca
Updated Branches:
  refs/heads/ARIA-30-SQL-based-storage-implementation f68e11bfb -> fe944e58d 
(forced update)


http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/fe944e58/tests/orchestrator/workflows/api/test_task.py
--
diff --git a/tests/orchestrator/workflows/api/test_task.py 
b/tests/orchestrator/workflows/api/test_task.py
index 8536902..4da42c1 100644
--- a/tests/orchestrator/workflows/api/test_task.py
+++ b/tests/orchestrator/workflows/api/test_task.py
@@ -22,7 +22,7 @@ from aria.orchestrator.workflows import api
 from tests import mock
 
 
-@pytest.fixture()
+@pytest.fixture
 def ctx():
 """
 Create the following graph in storage:
@@ -30,50 +30,26 @@ def ctx():
 :return:
 """
 simple_context = mock.context.simple()
-dependency_node = mock.models.get_dependency_node()
-dependency_node_instance = mock.models.get_dependency_node_instance(
-dependency_node=dependency_node)
-
-relationship = mock.models.get_relationship(dependency_node)
-relationship_instance = mock.models.get_relationship_instance(
-relationship=relationship,
-target_instance=dependency_node_instance
-)
-
-dependent_node = mock.models.get_dependent_node(relationship)
-dependent_node_instance = mock.models.get_dependent_node_instance(
-dependent_node=dependent_node,
-relationship_instance=relationship_instance
-)
-
-simple_context.model.node.store(dependent_node)
-simple_context.model.node.store(dependency_node)
-simple_context.model.node_instance.store(dependent_node_instance)
-simple_context.model.node_instance.store(dependency_node_instance)
-simple_context.model.relationship.store(relationship)
-simple_context.model.relationship_instance.store(relationship_instance)
-simple_context.model.execution.store(mock.models.get_execution())
-simple_context.model.deployment.store(mock.models.get_deployment())
+
simple_context.model.execution.store(mock.models.get_execution(simple_context.deployment))
 
 return simple_context
 
 
 class TestOperationTask(object):
 
-def test_node_operation_task_creation(self):
-workflow_context = mock.context.simple()
-
+def test_node_operation_task_creation(self, ctx):
 operation_name = 'aria.interfaces.lifecycle.create'
 op_details = {'operation': True}
-node = mock.models.get_dependency_node()
+node = ctx.model.node.get(mock.models.DEPENDENT_NODE_ID)
 node.operations[operation_name] = op_details
-node_instance = 
mock.models.get_dependency_node_instance(dependency_node=node)
+ctx.model.node.update(node)
+node_instance = 
ctx.model.node_instance.get(mock.models.DEPENDENT_NODE_INSTANCE_ID)
 inputs = {'inputs': True}
 max_attempts = 10
 retry_interval = 10
 ignore_failure = True
 
-with context.workflow.current.push(workflow_context):
+with context.workflow.current.push(ctx):
 api_task = api.task.OperationTask.node_instance(
 name=operation_name,
 instance=node_instance,
@@ -90,19 +66,18 @@ class TestOperationTask(object):
 assert api_task.max_attempts == max_attempts
 assert api_task.ignore_failure == ignore_failure
 
-def test_relationship_operation_task_creation(self):
-workflow_context = mock.context.simple()
-
+def test_relationship_operation_task_creation(self, ctx):
 operation_name = 'aria.interfaces.relationship_lifecycle.preconfigure'
 op_details = {'operation': True}
-relationship = mock.models.get_relationship()
+relationship = ctx.model.relationship.get(mock.models.RELATIONSHIP_ID)
 relationship.source_operations[operation_name] = op_details
-relationship_instance = 
mock.models.get_relationship_instance(relationship=relationship)
+relationship_instance = ctx.model.relationship_instance.get(
+mock.models.RELATIONSHIP_INSTANCE_ID)
 inputs = {'inputs': True}
 max_attempts = 10
 retry_interval = 10
 
-with context.workflow.current.push(workflow_context):
+with context.workflow.current.push(ctx):
 api_task = api.task.OperationTask.relationship_instance(
 name=operation_name,
 instance=relationship_instance,
@@ -118,18 +93,19 @@ class TestOperationTask(object):
 assert api_task.retry_interval == retry_interval
 assert api_task.max_attempts == max_attempts
 
-def test_operation_task_default_values(self):
-workflow_context = mock.context.simple(task_ignore_failure=True)
-with context.workflow.current.push(workflow_context):
-model_task = api.task.OperationTask(
+def test_operation_task_default_values(self, ctx):
+dependency_node_instance = 

[1/3] incubator-ariatosca git commit: ARIA-30-SQL-based-storage-implementation [Forced Update!]

[mxmrlv]

Repository: incubator-ariatosca
Updated Branches:
  refs/heads/ARIA-30-SQL-based-storage-implementation 88bc5d180 -> f68e11bfb 
(forced update)


http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/f68e11bf/tests/orchestrator/workflows/api/test_task.py
--
diff --git a/tests/orchestrator/workflows/api/test_task.py 
b/tests/orchestrator/workflows/api/test_task.py
index 8536902..4da42c1 100644
--- a/tests/orchestrator/workflows/api/test_task.py
+++ b/tests/orchestrator/workflows/api/test_task.py
@@ -22,7 +22,7 @@ from aria.orchestrator.workflows import api
 from tests import mock
 
 
-@pytest.fixture()
+@pytest.fixture
 def ctx():
 """
 Create the following graph in storage:
@@ -30,50 +30,26 @@ def ctx():
 :return:
 """
 simple_context = mock.context.simple()
-dependency_node = mock.models.get_dependency_node()
-dependency_node_instance = mock.models.get_dependency_node_instance(
-dependency_node=dependency_node)
-
-relationship = mock.models.get_relationship(dependency_node)
-relationship_instance = mock.models.get_relationship_instance(
-relationship=relationship,
-target_instance=dependency_node_instance
-)
-
-dependent_node = mock.models.get_dependent_node(relationship)
-dependent_node_instance = mock.models.get_dependent_node_instance(
-dependent_node=dependent_node,
-relationship_instance=relationship_instance
-)
-
-simple_context.model.node.store(dependent_node)
-simple_context.model.node.store(dependency_node)
-simple_context.model.node_instance.store(dependent_node_instance)
-simple_context.model.node_instance.store(dependency_node_instance)
-simple_context.model.relationship.store(relationship)
-simple_context.model.relationship_instance.store(relationship_instance)
-simple_context.model.execution.store(mock.models.get_execution())
-simple_context.model.deployment.store(mock.models.get_deployment())
+
simple_context.model.execution.store(mock.models.get_execution(simple_context.deployment))
 
 return simple_context
 
 
 class TestOperationTask(object):
 
-def test_node_operation_task_creation(self):
-workflow_context = mock.context.simple()
-
+def test_node_operation_task_creation(self, ctx):
 operation_name = 'aria.interfaces.lifecycle.create'
 op_details = {'operation': True}
-node = mock.models.get_dependency_node()
+node = ctx.model.node.get(mock.models.DEPENDENT_NODE_ID)
 node.operations[operation_name] = op_details
-node_instance = 
mock.models.get_dependency_node_instance(dependency_node=node)
+ctx.model.node.update(node)
+node_instance = 
ctx.model.node_instance.get(mock.models.DEPENDENT_NODE_INSTANCE_ID)
 inputs = {'inputs': True}
 max_attempts = 10
 retry_interval = 10
 ignore_failure = True
 
-with context.workflow.current.push(workflow_context):
+with context.workflow.current.push(ctx):
 api_task = api.task.OperationTask.node_instance(
 name=operation_name,
 instance=node_instance,
@@ -90,19 +66,18 @@ class TestOperationTask(object):
 assert api_task.max_attempts == max_attempts
 assert api_task.ignore_failure == ignore_failure
 
-def test_relationship_operation_task_creation(self):
-workflow_context = mock.context.simple()
-
+def test_relationship_operation_task_creation(self, ctx):
 operation_name = 'aria.interfaces.relationship_lifecycle.preconfigure'
 op_details = {'operation': True}
-relationship = mock.models.get_relationship()
+relationship = ctx.model.relationship.get(mock.models.RELATIONSHIP_ID)
 relationship.source_operations[operation_name] = op_details
-relationship_instance = 
mock.models.get_relationship_instance(relationship=relationship)
+relationship_instance = ctx.model.relationship_instance.get(
+mock.models.RELATIONSHIP_INSTANCE_ID)
 inputs = {'inputs': True}
 max_attempts = 10
 retry_interval = 10
 
-with context.workflow.current.push(workflow_context):
+with context.workflow.current.push(ctx):
 api_task = api.task.OperationTask.relationship_instance(
 name=operation_name,
 instance=relationship_instance,
@@ -118,18 +93,19 @@ class TestOperationTask(object):
 assert api_task.retry_interval == retry_interval
 assert api_task.max_attempts == max_attempts
 
-def test_operation_task_default_values(self):
-workflow_context = mock.context.simple(task_ignore_failure=True)
-with context.workflow.current.push(workflow_context):
-model_task = api.task.OperationTask(
+def test_operation_task_default_values(self, ctx):
+dependency_node_instance = 

[6/6] incubator-ariatosca git commit: Storage is now sql based with SQLAlchemy based models

[mxmrlv]

Storage is now sql based with SQLAlchemy based models


Project: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/commit/88bc5d18
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/tree/88bc5d18
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/diff/88bc5d18

Branch: refs/heads/ARIA-30-SQL-based-storage-implementation
Commit: 88bc5d18037023eaa466a81cb883b3b14d44335e
Parents: fe974e4
Author: mxmrlv 
Authored: Sun Nov 27 13:20:46 2016 +0200
Committer: mxmrlv 
Committed: Thu Dec 1 14:35:10 2016 +0200

--
 aria/__init__.py|  43 +-
 aria/orchestrator/__init__.py   |   4 +-
 aria/orchestrator/context/common.py |   2 +-
 aria/orchestrator/context/exceptions.py |   4 +-
 aria/orchestrator/context/operation.py  |   8 +-
 aria/orchestrator/context/toolbelt.py   |  13 +-
 aria/orchestrator/context/workflow.py   |  20 +-
 aria/orchestrator/exceptions.py |   7 +-
 aria/orchestrator/workflows/api/task.py |  10 +-
 aria/orchestrator/workflows/builtin/heal.py |  25 +-
 aria/orchestrator/workflows/builtin/install.py  |   7 +-
 .../orchestrator/workflows/builtin/uninstall.py |   5 +-
 .../orchestrator/workflows/builtin/workflows.py |   4 +-
 aria/orchestrator/workflows/core/task.py|  21 +-
 aria/storage/__init__.py| 379 ++--
 aria/storage/api.py | 219 +
 aria/storage/drivers.py | 416 -
 aria/storage/exceptions.py  |   4 +-
 aria/storage/filesystem_api.py  |  39 +
 aria/storage/mapi/__init__.py   |  20 +
 aria/storage/mapi/filesystem.py | 118 +++
 aria/storage/mapi/inmemory.py   | 148 +++
 aria/storage/mapi/sql.py| 369 
 aria/storage/models.py  | 912 +--
 aria/storage/rapi/__init__.py   |  18 +
 aria/storage/rapi/filesystem.py | 119 +++
 aria/storage/structures.py  | 424 -
 requirements.txt|   1 +
 tests/mock/context.py   |  50 +-
 tests/mock/models.py|  68 +-
 tests/orchestrator/context/test_operation.py|  36 +-
 tests/orchestrator/context/test_toolbelt.py |  47 +-
 tests/orchestrator/context/test_workflow.py |  10 +-
 tests/orchestrator/workflows/api/test_task.py   |  68 +-
 .../orchestrator/workflows/builtin/__init__.py  |  35 +-
 .../workflows/builtin/test_execute_operation.py |  11 +-
 .../orchestrator/workflows/builtin/test_heal.py |  18 +-
 .../workflows/builtin/test_install.py   |  14 +-
 .../workflows/builtin/test_uninstall.py |  12 +-
 .../orchestrator/workflows/core/test_engine.py  |  71 +-
 tests/orchestrator/workflows/core/test_task.py  |  20 +-
 .../test_task_graph_into_exececution_graph.py   |  10 +-
 tests/requirements.txt  |   2 +-
 tests/storage/__init__.py   |  38 +-
 tests/storage/test_drivers.py   | 135 ---
 tests/storage/test_field.py | 124 ---
 tests/storage/test_model_storage.py | 167 ++--
 tests/storage/test_models.py| 364 
 tests/storage/test_models_api.py|  70 --
 tests/storage/test_resource_storage.py  |  57 +-
 50 files changed, 2318 insertions(+), 2468 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/88bc5d18/aria/__init__.py
--
diff --git a/aria/__init__.py b/aria/__init__.py
index 3f81f98..6e810f0 100644
--- a/aria/__init__.py
+++ b/aria/__init__.py
@@ -23,7 +23,6 @@ import pkgutil
 from .VERSION import version as __version__
 
 from .orchestrator.decorators import workflow, operation
-from .storage import ModelStorage, ResourceStorage, models, ModelDriver, 
ResourceDriver
 from . import (
 utils,
 parser,
@@ -58,37 +57,37 @@ def install_aria_extensions():
 del sys.modules[module_name]
 
 
-def application_model_storage(driver):
+def application_model_storage(api, api_params=None):
 """
 Initiate model storage for the supplied storage driver
 """
-
-assert isinstance(driver, ModelDriver)
-if driver not in _model_storage:
-_model_storage[driver] = ModelStorage(
-driver, model_classes=[
-models.Node,
-models.NodeInstance,
-models.Plugin,
-models.Blueprint,
-models.Snapshot,
-

[3/6] incubator-ariatosca git commit: ARIA-28 Integrate with appveyor

[mxmrlv]

ARIA-28 Integrate with appveyor


Project: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/commit/fe974e49
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/tree/fe974e49
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/diff/fe974e49

Branch: refs/heads/ARIA-30-SQL-based-storage-implementation
Commit: fe974e49f7e209dce9eb252c67406b02509bd0b5
Parents: d7addbc
Author: Dan Kilman 
Authored: Wed Nov 30 14:13:06 2016 +0200
Committer: Dan Kilman 
Committed: Wed Nov 30 15:29:27 2016 +0200

--
 appveyor.yml  | 26 +
 tests/storage/__init__.py |  2 +-
 tests/test_logger.py  | 63 +-
 tox.ini   |  7 +++--
 4 files changed, 63 insertions(+), 35 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/fe974e49/appveyor.yml
--
diff --git a/appveyor.yml b/appveyor.yml
new file mode 100644
index 000..3ea8635
--- /dev/null
+++ b/appveyor.yml
@@ -0,0 +1,26 @@
+environment:
+
+  TOX_ENV: pywin
+
+  matrix:
+- PYTHON: C:\Python27
+  PYTHON_VERSION: 2.7.8
+  PYTHON_ARCH: 32
+
+build: false
+
+install:
+  - SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%
+  - ps: (new-object 
System.Net.WebClient).Downloadfile('https://bootstrap.pypa.io/get-pip.py', 
'C:\Users\appveyor\get-pip.py')
+  - ps: Start-Process -FilePath "C:\Python27\python.exe" -ArgumentList 
"C:\Users\appveyor\get-pip.py" -Wait -Passthru
+
+before_test:
+  - pip install virtualenv --upgrade
+  - virtualenv env
+  - 'env\Scripts\activate.bat'
+  - pip install tox
+
+test_script:
+  - pip --version
+  - tox --version
+  - tox -e %TOX_ENV%

http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/fe974e49/tests/storage/__init__.py
--
diff --git a/tests/storage/__init__.py b/tests/storage/__init__.py
index 3408f2b..9bf48cc 100644
--- a/tests/storage/__init__.py
+++ b/tests/storage/__init__.py
@@ -50,4 +50,4 @@ class TestFileSystem(object):
 self.path = mkdtemp('{0}'.format(self.__class__.__name__))
 
 def teardown_method(self):
-rmtree(self.path)
+rmtree(self.path, ignore_errors=True)

http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/fe974e49/tests/test_logger.py
--
diff --git a/tests/test_logger.py b/tests/test_logger.py
index 8c7a9af..0199068 100644
--- a/tests/test_logger.py
+++ b/tests/test_logger.py
@@ -14,7 +14,6 @@
 # limitations under the License.
 
 import logging
-import tempfile
 
 from aria.logger import (create_logger,
  create_console_log_handler,
@@ -70,40 +69,40 @@ def test_create_console_log_handler(capsys):
 assert err.count(info_test_string) == 1
 
 
-def test_create_file_log_handler():
+def test_create_file_log_handler(tmpdir):
 
 test_string = 'create_file_log_test_string'
 
-with tempfile.NamedTemporaryFile() as temp_file:
-handler = create_file_log_handler(file_path=temp_file.name)
-assert handler.baseFilename == temp_file.name
-assert handler.maxBytes == 5 * 1000 * 1024
-assert handler.backupCount == 10
-assert handler.stream is None
-assert handler.level == logging.DEBUG
-assert handler.formatter == _default_file_formatter
-
-logger = create_logger(handlers=[handler])
-logger.debug(test_string)
-assert test_string in temp_file.read()
-
-with tempfile.NamedTemporaryFile() as temp_file:
-handler = create_file_log_handler(
-file_path=temp_file.name,
-level=logging.INFO,
-max_bytes=1000,
-backup_count=2,
-formatter=logging.Formatter()
-)
-assert handler.baseFilename == temp_file.name
-assert handler.level == logging.INFO
-assert handler.maxBytes == 1000
-assert handler.backupCount == 2
-assert isinstance(handler.formatter, logging.Formatter)
-
-logger = create_logger(handlers=[handler])
-logger.info(test_string)
-assert test_string in temp_file.read()
+debug_log = tmpdir.join('debug.log')
+handler = create_file_log_handler(file_path=str(debug_log))
+assert handler.baseFilename == str(debug_log)
+assert handler.maxBytes == 5 * 1000 * 1024
+assert handler.backupCount == 10
+assert handler.stream is None
+assert handler.level == logging.DEBUG
+assert handler.formatter == _default_file_formatter
+
+logger = create_logger(handlers=[handler])
+logger.debug(test_string)
+assert test_string in 

[incubator-ariatosca] Git Push Summary

[mxmrlv]

Repository: incubator-ariatosca
Updated Branches:
  refs/heads/SQLAlchemy-based-models [deleted] 2d6b9375d

[1/3] incubator-ariatosca git commit: Storage is now sql based with SQLAlchemy based models [Forced Update!]

[mxmrlv]

Repository: incubator-ariatosca
Updated Branches:
  refs/heads/ARIA-30-SQL-based-storage-implementation 91f9de43b -> 8d768e611 
(forced update)


http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/8d768e61/tests/orchestrator/workflows/api/test_task.py
--
diff --git a/tests/orchestrator/workflows/api/test_task.py 
b/tests/orchestrator/workflows/api/test_task.py
index 8536902..4da42c1 100644
--- a/tests/orchestrator/workflows/api/test_task.py
+++ b/tests/orchestrator/workflows/api/test_task.py
@@ -22,7 +22,7 @@ from aria.orchestrator.workflows import api
 from tests import mock
 
 
-@pytest.fixture()
+@pytest.fixture
 def ctx():
 """
 Create the following graph in storage:
@@ -30,50 +30,26 @@ def ctx():
 :return:
 """
 simple_context = mock.context.simple()
-dependency_node = mock.models.get_dependency_node()
-dependency_node_instance = mock.models.get_dependency_node_instance(
-dependency_node=dependency_node)
-
-relationship = mock.models.get_relationship(dependency_node)
-relationship_instance = mock.models.get_relationship_instance(
-relationship=relationship,
-target_instance=dependency_node_instance
-)
-
-dependent_node = mock.models.get_dependent_node(relationship)
-dependent_node_instance = mock.models.get_dependent_node_instance(
-dependent_node=dependent_node,
-relationship_instance=relationship_instance
-)
-
-simple_context.model.node.store(dependent_node)
-simple_context.model.node.store(dependency_node)
-simple_context.model.node_instance.store(dependent_node_instance)
-simple_context.model.node_instance.store(dependency_node_instance)
-simple_context.model.relationship.store(relationship)
-simple_context.model.relationship_instance.store(relationship_instance)
-simple_context.model.execution.store(mock.models.get_execution())
-simple_context.model.deployment.store(mock.models.get_deployment())
+
simple_context.model.execution.store(mock.models.get_execution(simple_context.deployment))
 
 return simple_context
 
 
 class TestOperationTask(object):
 
-def test_node_operation_task_creation(self):
-workflow_context = mock.context.simple()
-
+def test_node_operation_task_creation(self, ctx):
 operation_name = 'aria.interfaces.lifecycle.create'
 op_details = {'operation': True}
-node = mock.models.get_dependency_node()
+node = ctx.model.node.get(mock.models.DEPENDENT_NODE_ID)
 node.operations[operation_name] = op_details
-node_instance = 
mock.models.get_dependency_node_instance(dependency_node=node)
+ctx.model.node.update(node)
+node_instance = 
ctx.model.node_instance.get(mock.models.DEPENDENT_NODE_INSTANCE_ID)
 inputs = {'inputs': True}
 max_attempts = 10
 retry_interval = 10
 ignore_failure = True
 
-with context.workflow.current.push(workflow_context):
+with context.workflow.current.push(ctx):
 api_task = api.task.OperationTask.node_instance(
 name=operation_name,
 instance=node_instance,
@@ -90,19 +66,18 @@ class TestOperationTask(object):
 assert api_task.max_attempts == max_attempts
 assert api_task.ignore_failure == ignore_failure
 
-def test_relationship_operation_task_creation(self):
-workflow_context = mock.context.simple()
-
+def test_relationship_operation_task_creation(self, ctx):
 operation_name = 'aria.interfaces.relationship_lifecycle.preconfigure'
 op_details = {'operation': True}
-relationship = mock.models.get_relationship()
+relationship = ctx.model.relationship.get(mock.models.RELATIONSHIP_ID)
 relationship.source_operations[operation_name] = op_details
-relationship_instance = 
mock.models.get_relationship_instance(relationship=relationship)
+relationship_instance = ctx.model.relationship_instance.get(
+mock.models.RELATIONSHIP_INSTANCE_ID)
 inputs = {'inputs': True}
 max_attempts = 10
 retry_interval = 10
 
-with context.workflow.current.push(workflow_context):
+with context.workflow.current.push(ctx):
 api_task = api.task.OperationTask.relationship_instance(
 name=operation_name,
 instance=relationship_instance,
@@ -118,18 +93,19 @@ class TestOperationTask(object):
 assert api_task.retry_interval == retry_interval
 assert api_task.max_attempts == max_attempts
 
-def test_operation_task_default_values(self):
-workflow_context = mock.context.simple(task_ignore_failure=True)
-with context.workflow.current.push(workflow_context):
-model_task = api.task.OperationTask(
+def test_operation_task_default_values(self, ctx):
+dependency_node_instance = 

[2/3] incubator-ariatosca git commit: Storage is now sql based with SQLAlchemy based models

[mxmrlv]

http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/8d768e61/aria/storage/mapi/sql.py
--
diff --git a/aria/storage/mapi/sql.py b/aria/storage/mapi/sql.py
new file mode 100644
index 000..4408aa3
--- /dev/null
+++ b/aria/storage/mapi/sql.py
@@ -0,0 +1,369 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+SQLalchemy based MAPI
+"""
+
+from sqlite3 import DatabaseError as SQLiteDBError
+from sqlalchemy.exc import SQLAlchemyError
+from sqlalchemy.sql.elements import Label
+
+from aria.utils.collections import OrderedDict
+
+
+try:
+from psycopg2 import DatabaseError as Psycopg2DBError
+sql_errors = (SQLAlchemyError, SQLiteDBError, Psycopg2DBError)
+except ImportError:
+sql_errors = (SQLAlchemyError, SQLiteDBError)
+Psycopg2DBError = None
+
+from ... import storage
+
+
+DEFAULT_SQL_DIALECT = 'sqlite'
+
+
+class SQLAlchemyModelAPI(storage.api.ModelAPI):
+"""
+SQL based MAPI.
+"""
+
+def __init__(self,
+ engine,
+ session,
+ **kwargs):
+super(SQLAlchemyModelAPI, self).__init__(**kwargs)
+self._engine = engine
+self._session = session
+
+def get(self, entry_id, include=None, filters=None, locking=False, 
**kwargs):
+"""Return a single result based on the model class and element ID
+"""
+filters = filters or {'id': entry_id}
+query = self._get_query(include, filters)
+if locking:
+query = query.with_for_update()
+result = query.first()
+
+if not result:
+raise storage.exceptions.StorageError(
+'Requested {0} with ID `{1}` was not found'
+.format(self.model_cls.__name__, entry_id)
+)
+return result
+
+def iter(self,
+ include=None,
+ filters=None,
+ pagination=None,
+ sort=None,
+ **kwargs):
+"""Return a (possibly empty) list of `model_class` results
+"""
+query = self._get_query(include, filters, sort)
+
+results, _, _, _ = self._paginate(query, pagination)
+
+for result in results:
+yield result
+
+def store(self, entry, **kwargs):
+"""Create a `model_class` instance from a serializable `model` object
+
+:param entry: A dict with relevant kwargs, or an instance of a class
+that has a `to_dict` method, and whose attributes match the columns
+of `model_class` (might also my just an instance of `model_class`)
+:return: An instance of `model_class`
+"""
+self._session.add(entry)
+self._safe_commit()
+return entry
+
+def delete(self, entry_id, filters=None, **kwargs):
+"""Delete a single result based on the model class and element ID
+"""
+try:
+instance = self.get(
+entry_id,
+filters=filters
+)
+except storage.exceptions.StorageError:
+raise storage.exceptions.StorageError(
+'Could not delete {0} with ID `{1}` - element not found'
+.format(
+self.model_cls.__name__,
+entry_id
+)
+)
+self._load_properties(instance)
+self._session.delete(instance)
+self._safe_commit()
+return instance
+
+# TODO: this might need rework
+def update(self, entry, **kwargs):
+"""Add `instance` to the DB session, and attempt to commit
+
+:return: The updated instance
+"""
+return self.store(entry)
+
+def refresh(self, entry):
+"""Reload the instance with fresh information from the DB
+
+:param entry: Instance to be re-loaded from the DB
+:return: The refreshed instance
+"""
+self._session.refresh(entry)
+self._load_properties(entry)
+return entry
+
+def _destroy_connection(self):
+pass
+
+def _establish_connection(self):
+pass
+
+def create(self):
+self.model_cls.__table__.create(self._engine)
+
+def drop(self):
+"""
+  

[3/3] incubator-ariatosca git commit: Storage is now sql based with SQLAlchemy based models

[mxmrlv]

Storage is now sql based with SQLAlchemy based models


Project: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/commit/8d768e61
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/tree/8d768e61
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/diff/8d768e61

Branch: refs/heads/ARIA-30-SQL-based-storage-implementation
Commit: 8d768e61147910438f71ad74627a61f1d617e56e
Parents: b33c70e
Author: mxmrlv 
Authored: Sun Nov 27 13:20:46 2016 +0200
Committer: mxmrlv 
Committed: Thu Dec 1 12:58:43 2016 +0200

--
 aria/__init__.py|  43 +-
 aria/orchestrator/__init__.py   |   4 +-
 aria/orchestrator/context/common.py |   2 +-
 aria/orchestrator/context/exceptions.py |   4 +-
 aria/orchestrator/context/operation.py  |   8 +-
 aria/orchestrator/context/toolbelt.py   |  13 +-
 aria/orchestrator/context/workflow.py   |  20 +-
 aria/orchestrator/exceptions.py |   7 +-
 aria/orchestrator/workflows/api/task.py |  10 +-
 aria/orchestrator/workflows/builtin/heal.py |  25 +-
 aria/orchestrator/workflows/builtin/install.py  |   7 +-
 .../orchestrator/workflows/builtin/uninstall.py |   5 +-
 .../orchestrator/workflows/builtin/workflows.py |   4 +-
 aria/orchestrator/workflows/core/task.py|  21 +-
 aria/storage/__init__.py| 379 ++--
 aria/storage/api.py | 219 +
 aria/storage/drivers.py | 416 -
 aria/storage/exceptions.py  |   4 +-
 aria/storage/filesystem_api.py  |  39 +
 aria/storage/mapi/__init__.py   |  20 +
 aria/storage/mapi/filesystem.py | 118 +++
 aria/storage/mapi/inmemory.py   | 148 +++
 aria/storage/mapi/sql.py| 369 
 aria/storage/models.py  | 912 +--
 aria/storage/rapi/__init__.py   |  18 +
 aria/storage/rapi/filesystem.py | 119 +++
 aria/storage/structures.py  | 424 -
 requirements.txt|   1 +
 tests/mock/context.py   |  50 +-
 tests/mock/models.py|  68 +-
 tests/orchestrator/context/test_operation.py|  36 +-
 tests/orchestrator/context/test_toolbelt.py |  47 +-
 tests/orchestrator/context/test_workflow.py |  10 +-
 tests/orchestrator/workflows/api/test_task.py   |  68 +-
 .../orchestrator/workflows/builtin/__init__.py  |  35 +-
 .../workflows/builtin/test_execute_operation.py |  11 +-
 .../orchestrator/workflows/builtin/test_heal.py |  18 +-
 .../workflows/builtin/test_install.py   |  14 +-
 .../workflows/builtin/test_uninstall.py |  12 +-
 .../orchestrator/workflows/core/test_engine.py  |  71 +-
 tests/orchestrator/workflows/core/test_task.py  |  20 +-
 .../test_task_graph_into_exececution_graph.py   |  10 +-
 tests/requirements.txt  |   2 +-
 tests/storage/__init__.py   |  41 +-
 tests/storage/test_drivers.py   | 135 ---
 tests/storage/test_field.py | 124 ---
 tests/storage/test_model_storage.py | 167 ++--
 tests/storage/test_models.py| 364 
 tests/storage/test_models_api.py|  70 --
 tests/storage/test_resource_storage.py  |  57 +-
 50 files changed, 2319 insertions(+), 2470 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/8d768e61/aria/__init__.py
--
diff --git a/aria/__init__.py b/aria/__init__.py
index 3f81f98..6e810f0 100644
--- a/aria/__init__.py
+++ b/aria/__init__.py
@@ -23,7 +23,6 @@ import pkgutil
 from .VERSION import version as __version__
 
 from .orchestrator.decorators import workflow, operation
-from .storage import ModelStorage, ResourceStorage, models, ModelDriver, 
ResourceDriver
 from . import (
 utils,
 parser,
@@ -58,37 +57,37 @@ def install_aria_extensions():
 del sys.modules[module_name]
 
 
-def application_model_storage(driver):
+def application_model_storage(api, api_params=None):
 """
 Initiate model storage for the supplied storage driver
 """
-
-assert isinstance(driver, ModelDriver)
-if driver not in _model_storage:
-_model_storage[driver] = ModelStorage(
-driver, model_classes=[
-models.Node,
-models.NodeInstance,
-models.Plugin,
-models.Blueprint,
-models.Snapshot,
-

[GitHub] incubator-ariatosca pull request #31: Aria 30 sql based storage implementati...

[mxmrlv]

GitHub user mxmrlv opened a pull request:

https://github.com/apache/incubator-ariatosca/pull/31

Aria 30 sql based storage implementation



You can merge this pull request into a Git repository by running:

$ git pull https://github.com/apache/incubator-ariatosca 
ARIA-30-SQL-based-storage-implementation

Alternatively you can review and apply these changes as the patch at:

https://github.com/apache/incubator-ariatosca/pull/31.patch

To close this pull request, make a commit to your master/trunk branch
with (at least) the following in the commit message:

This closes #31


commit 2d6b9375d0665f4fabbd2acac8d5f10a0d20adcb
Author: mxmrlv 
Date:   2016-11-27T11:20:46Z

Storage is now sql based with SQLAlchemy based models

commit 91f9de43b0042d5619f9d9539350fc5847287be6
Author: mxmrlv 
Date:   2016-12-01T10:56:11Z

more lynting




---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

incubator-ariatosca git commit: more lynting [Forced Update!]

[mxmrlv]

Repository: incubator-ariatosca
Updated Branches:
  refs/heads/ARIA-30-SQL-based-storage-implementation aaf35c1d2 -> 91f9de43b 
(forced update)


more lynting


Project: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/commit/91f9de43
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/tree/91f9de43
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/diff/91f9de43

Branch: refs/heads/ARIA-30-SQL-based-storage-implementation
Commit: 91f9de43b0042d5619f9d9539350fc5847287be6
Parents: 2d6b937
Author: mxmrlv 
Authored: Thu Dec 1 12:56:11 2016 +0200
Committer: mxmrlv 
Committed: Thu Dec 1 12:58:13 2016 +0200

--
 aria/__init__.py| 1 -
 aria/storage/filesystem_api.py  | 4 
 aria/storage/mapi/sql.py| 7 ---
 aria/storage/models.py  | 2 +-
 aria/storage/rapi/filesystem.py | 6 +++---
 aria/storage/structures.py  | 7 +++
 tests/requirements.txt  | 3 +--
 7 files changed, 16 insertions(+), 14 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/91f9de43/aria/__init__.py
--
diff --git a/aria/__init__.py b/aria/__init__.py
index 2c00729..6e810f0 100644
--- a/aria/__init__.py
+++ b/aria/__init__.py
@@ -23,7 +23,6 @@ import pkgutil
 from .VERSION import version as __version__
 
 from .orchestrator.decorators import workflow, operation
-from . import storage
 from . import (
 utils,
 parser,

http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/91f9de43/aria/storage/filesystem_api.py
--
diff --git a/aria/storage/filesystem_api.py b/aria/storage/filesystem_api.py
index d42cd61..f28d1f6 100644
--- a/aria/storage/filesystem_api.py
+++ b/aria/storage/filesystem_api.py
@@ -24,6 +24,10 @@ class BaseFileSystemAPI(api.StorageAPI):
 """
 Base class which handles storage on the file system.
 """
+
+def create(self, **kwargs):
+super(BaseFileSystemAPI, self).create(**kwargs)
+
 def __init__(self, *args, **kwargs):
 super(BaseFileSystemAPI, self).__init__(*args, **kwargs)
 self._lock = RLock()

http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/91f9de43/aria/storage/mapi/sql.py
--
diff --git a/aria/storage/mapi/sql.py b/aria/storage/mapi/sql.py
index f2ec0e5..4408aa3 100644
--- a/aria/storage/mapi/sql.py
+++ b/aria/storage/mapi/sql.py
@@ -49,7 +49,7 @@ class SQLAlchemyModelAPI(storage.api.ModelAPI):
 self._engine = engine
 self._session = session
 
-def get(self, entry_id, include=None, filters=None, locking=False):
+def get(self, entry_id, include=None, filters=None, locking=False, 
**kwargs):
 """Return a single result based on the model class and element ID
 """
 filters = filters or {'id': entry_id}
@@ -69,7 +69,8 @@ class SQLAlchemyModelAPI(storage.api.ModelAPI):
  include=None,
  filters=None,
  pagination=None,
- sort=None):
+ sort=None,
+ **kwargs):
 """Return a (possibly empty) list of `model_class` results
 """
 query = self._get_query(include, filters, sort)
@@ -91,7 +92,7 @@ class SQLAlchemyModelAPI(storage.api.ModelAPI):
 self._safe_commit()
 return entry
 
-def delete(self, entry_id, filters=None):
+def delete(self, entry_id, filters=None, **kwargs):
 """Delete a single result based on the model class and element ID
 """
 try:

http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/91f9de43/aria/storage/models.py
--
diff --git a/aria/storage/models.py b/aria/storage/models.py
index 9d0515e..c04f7d8 100644
--- a/aria/storage/models.py
+++ b/aria/storage/models.py
@@ -318,7 +318,7 @@ class DeploymentUpdate(SQLModelBase):
 """
 return self.deployment.id
 
-def to_dict(self, suppress_error=False):
+def to_dict(self, suppress_error=False, **kwargs):
 dep_update_dict = super(DeploymentUpdate, self).to_dict(suppress_error)
 # Taking care of the fact the DeploymentSteps are objects
 dep_update_dict['steps'] = [step.to_dict() for step in self.steps]

http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/91f9de43/aria/storage/rapi/filesystem.py
--
diff --git a/aria/storage/rapi/filesystem.py b/aria/storage/rapi/filesystem.py
index ceafde5..a6c4ddf 100644
--- a/aria/storage/rapi/filesystem.py
+++ 

incubator-ariatosca git commit: more lynting

[mxmrlv]

Repository: incubator-ariatosca
Updated Branches:
  refs/heads/ARIA-30-SQL-based-storage-implementation 2d6b9375d -> aaf35c1d2


more lynting


Project: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/commit/aaf35c1d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/tree/aaf35c1d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/diff/aaf35c1d

Branch: refs/heads/ARIA-30-SQL-based-storage-implementation
Commit: aaf35c1d200687fc16fe886653b9aad4c1b0674f
Parents: 2d6b937
Author: mxmrlv 
Authored: Thu Dec 1 12:56:11 2016 +0200
Committer: mxmrlv 
Committed: Thu Dec 1 12:56:11 2016 +0200

--
 aria/__init__.py| 1 -
 aria/storage/filesystem_api.py  | 4 
 aria/storage/mapi/sql.py| 7 ---
 aria/storage/models.py  | 2 +-
 aria/storage/rapi/filesystem.py | 6 +++---
 aria/storage/structures.py  | 7 +++
 6 files changed, 15 insertions(+), 12 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/aaf35c1d/aria/__init__.py
--
diff --git a/aria/__init__.py b/aria/__init__.py
index 2c00729..6e810f0 100644
--- a/aria/__init__.py
+++ b/aria/__init__.py
@@ -23,7 +23,6 @@ import pkgutil
 from .VERSION import version as __version__
 
 from .orchestrator.decorators import workflow, operation
-from . import storage
 from . import (
 utils,
 parser,

http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/aaf35c1d/aria/storage/filesystem_api.py
--
diff --git a/aria/storage/filesystem_api.py b/aria/storage/filesystem_api.py
index d42cd61..f28d1f6 100644
--- a/aria/storage/filesystem_api.py
+++ b/aria/storage/filesystem_api.py
@@ -24,6 +24,10 @@ class BaseFileSystemAPI(api.StorageAPI):
 """
 Base class which handles storage on the file system.
 """
+
+def create(self, **kwargs):
+super(BaseFileSystemAPI, self).create(**kwargs)
+
 def __init__(self, *args, **kwargs):
 super(BaseFileSystemAPI, self).__init__(*args, **kwargs)
 self._lock = RLock()

http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/aaf35c1d/aria/storage/mapi/sql.py
--
diff --git a/aria/storage/mapi/sql.py b/aria/storage/mapi/sql.py
index f2ec0e5..4408aa3 100644
--- a/aria/storage/mapi/sql.py
+++ b/aria/storage/mapi/sql.py
@@ -49,7 +49,7 @@ class SQLAlchemyModelAPI(storage.api.ModelAPI):
 self._engine = engine
 self._session = session
 
-def get(self, entry_id, include=None, filters=None, locking=False):
+def get(self, entry_id, include=None, filters=None, locking=False, 
**kwargs):
 """Return a single result based on the model class and element ID
 """
 filters = filters or {'id': entry_id}
@@ -69,7 +69,8 @@ class SQLAlchemyModelAPI(storage.api.ModelAPI):
  include=None,
  filters=None,
  pagination=None,
- sort=None):
+ sort=None,
+ **kwargs):
 """Return a (possibly empty) list of `model_class` results
 """
 query = self._get_query(include, filters, sort)
@@ -91,7 +92,7 @@ class SQLAlchemyModelAPI(storage.api.ModelAPI):
 self._safe_commit()
 return entry
 
-def delete(self, entry_id, filters=None):
+def delete(self, entry_id, filters=None, **kwargs):
 """Delete a single result based on the model class and element ID
 """
 try:

http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/aaf35c1d/aria/storage/models.py
--
diff --git a/aria/storage/models.py b/aria/storage/models.py
index 9d0515e..c04f7d8 100644
--- a/aria/storage/models.py
+++ b/aria/storage/models.py
@@ -318,7 +318,7 @@ class DeploymentUpdate(SQLModelBase):
 """
 return self.deployment.id
 
-def to_dict(self, suppress_error=False):
+def to_dict(self, suppress_error=False, **kwargs):
 dep_update_dict = super(DeploymentUpdate, self).to_dict(suppress_error)
 # Taking care of the fact the DeploymentSteps are objects
 dep_update_dict['steps'] = [step.to_dict() for step in self.steps]

http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/aaf35c1d/aria/storage/rapi/filesystem.py
--
diff --git a/aria/storage/rapi/filesystem.py b/aria/storage/rapi/filesystem.py
index ceafde5..a6c4ddf 100644
--- a/aria/storage/rapi/filesystem.py
+++ b/aria/storage/rapi/filesystem.py
@@ -21,10 +21,10 @@ from distutils 

[incubator-ariatosca] Git Push Summary

[mxmrlv]

Repository: incubator-ariatosca
Updated Branches:
  refs/heads/ARIA-25-Storage-genericizing [deleted] aa0aee206

[incubator-ariatosca] Git Push Summary

[mxmrlv]

Repository: incubator-ariatosca
Updated Branches:
  refs/heads/ARIA-30-SQL-based-storage-implementation [created] 2d6b9375d

[GitHub] incubator-ariatosca pull request #27: ARIA-25-Storage-genericizing

[mxmrlv]

Github user mxmrlv closed the pull request at:

https://github.com/apache/incubator-ariatosca/pull/27


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[jira] [Closed] (ARIATOSCA-25) Storage genericizing

[Maxim Orlov (JIRA)]

 [ 
https://issues.apache.org/jira/browse/ARIATOSCA-25?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
 ]

Maxim Orlov closed ARIATOSCA-25.

Resolution: Won't Fix

This is no longer relevant, as the storage has new architecture.

> Storage genericizing
> 
>
> Key: ARIATOSCA-25
> URL: https://issues.apache.org/jira/browse/ARIATOSCA-25
> Project: AriaTosca
>  Issue Type: Task
>Reporter: Maxim Orlov
>Assignee: Maxim Orlov
>
> Currently the storage handles pointers within in several different classes, 
> This must be changed so the driver handles all of the storage related 
> treatments (i.e. pointer resolution). 



--
This message was sent by Atlassian JIRA
(v6.3.4#6332)

Re: Podling Report Reminder - December 2016

[Arthur Berezin]

Podling for December updated:
https://cwiki.apache.org/confluence/display/ARIATOSCA/Podling+Report+2016-12

Arthur


On Wed, Nov 30, 2016 at 1:42 PM  wrote:

> Dear podling,
>
> This email was sent by an automated system on behalf of the Apache
> Incubator PMC. It is an initial reminder to give you plenty of time to
> prepare your quarterly board report.
>
> The board meeting is scheduled for Wed, 21 December 2016, 10:30 am PDT.
> The report for your podling will form a part of the Incubator PMC
> report. The Incubator PMC requires your report to be submitted 2 weeks
> before the board meeting, to allow sufficient time for review and
> submission (Wed, December 07).
>
> Please submit your report with sufficient time to allow the Incubator
> PMC, and subsequently board members to review and digest. Again, the
> very latest you should submit your report is 2 weeks prior to the board
> meeting.
>
> Thanks,
>
> The Apache Incubator PMC
>
> Submitting your Report
>
> --
>
> Your report should contain the following:
>
> *   Your project name
> *   A brief description of your project, which assumes no knowledge of
> the project or necessarily of its field
> *   A list of the three most important issues to address in the move
> towards graduation.
> *   Any issues that the Incubator PMC or ASF Board might wish/need to be
> aware of
> *   How has the community developed since the last report
> *   How has the project developed since the last report.
>
> This should be appended to the Incubator Wiki page at:
>
> https://wiki.apache.org/incubator/December2016
>
> Note: This is manually populated. You may need to wait a little before
> this page is created from a template.
>
> Mentors
> ---
>
> Mentors should review reports for their project(s) and sign them off on
> the Incubator wiki page. Signing off reports shows that you are
> following the project - projects that are not signed may raise alarms
> for the Incubator PMC.
>
> Incubator PMC
>

[3/3] incubator-ariatosca git commit: Storage is now sql based with SQLAlchemy based models

[mxmrlv]

Storage is now sql based with SQLAlchemy based models


Project: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/commit/2d6b9375
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/tree/2d6b9375
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/diff/2d6b9375

Branch: refs/heads/SQLAlchemy-based-models
Commit: 2d6b9375d0665f4fabbd2acac8d5f10a0d20adcb
Parents: b33c70e
Author: mxmrlv 
Authored: Sun Nov 27 13:20:46 2016 +0200
Committer: mxmrlv 
Committed: Thu Dec 1 12:32:21 2016 +0200

--
 aria/__init__.py|  44 +-
 aria/orchestrator/__init__.py   |   4 +-
 aria/orchestrator/context/common.py |   2 +-
 aria/orchestrator/context/exceptions.py |   4 +-
 aria/orchestrator/context/operation.py  |   8 +-
 aria/orchestrator/context/toolbelt.py   |  13 +-
 aria/orchestrator/context/workflow.py   |  20 +-
 aria/orchestrator/exceptions.py |   7 +-
 aria/orchestrator/workflows/api/task.py |  10 +-
 aria/orchestrator/workflows/builtin/heal.py |  25 +-
 aria/orchestrator/workflows/builtin/install.py  |   7 +-
 .../orchestrator/workflows/builtin/uninstall.py |   5 +-
 .../orchestrator/workflows/builtin/workflows.py |   4 +-
 aria/orchestrator/workflows/core/task.py|  21 +-
 aria/storage/__init__.py| 379 ++--
 aria/storage/api.py | 219 +
 aria/storage/drivers.py | 416 -
 aria/storage/exceptions.py  |   4 +-
 aria/storage/filesystem_api.py  |  35 +
 aria/storage/mapi/__init__.py   |  20 +
 aria/storage/mapi/filesystem.py | 118 +++
 aria/storage/mapi/inmemory.py   | 148 +++
 aria/storage/mapi/sql.py| 368 
 aria/storage/models.py  | 912 +--
 aria/storage/rapi/__init__.py   |  18 +
 aria/storage/rapi/filesystem.py | 119 +++
 aria/storage/structures.py  | 423 -
 requirements.txt|   1 +
 tests/mock/context.py   |  50 +-
 tests/mock/models.py|  68 +-
 tests/orchestrator/context/test_operation.py|  36 +-
 tests/orchestrator/context/test_toolbelt.py |  47 +-
 tests/orchestrator/context/test_workflow.py |  10 +-
 tests/orchestrator/workflows/api/test_task.py   |  68 +-
 .../orchestrator/workflows/builtin/__init__.py  |  35 +-
 .../workflows/builtin/test_execute_operation.py |  11 +-
 .../orchestrator/workflows/builtin/test_heal.py |  18 +-
 .../workflows/builtin/test_install.py   |  14 +-
 .../workflows/builtin/test_uninstall.py |  12 +-
 .../orchestrator/workflows/core/test_engine.py  |  71 +-
 tests/orchestrator/workflows/core/test_task.py  |  20 +-
 .../test_task_graph_into_exececution_graph.py   |  10 +-
 tests/requirements.txt  |   1 +
 tests/storage/__init__.py   |  41 +-
 tests/storage/test_drivers.py   | 135 ---
 tests/storage/test_field.py | 124 ---
 tests/storage/test_model_storage.py | 167 ++--
 tests/storage/test_models.py| 364 
 tests/storage/test_models_api.py|  70 --
 tests/storage/test_resource_storage.py  |  57 +-
 50 files changed, 2315 insertions(+), 2468 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/2d6b9375/aria/__init__.py
--
diff --git a/aria/__init__.py b/aria/__init__.py
index 3f81f98..2c00729 100644
--- a/aria/__init__.py
+++ b/aria/__init__.py
@@ -23,7 +23,7 @@ import pkgutil
 from .VERSION import version as __version__
 
 from .orchestrator.decorators import workflow, operation
-from .storage import ModelStorage, ResourceStorage, models, ModelDriver, 
ResourceDriver
+from . import storage
 from . import (
 utils,
 parser,
@@ -58,37 +58,37 @@ def install_aria_extensions():
 del sys.modules[module_name]
 
 
-def application_model_storage(driver):
+def application_model_storage(api, api_params=None):
 """
 Initiate model storage for the supplied storage driver
 """
-
-assert isinstance(driver, ModelDriver)
-if driver not in _model_storage:
-_model_storage[driver] = ModelStorage(
-driver, model_classes=[
-models.Node,
-models.NodeInstance,
-models.Plugin,
-models.Blueprint,
-models.Snapshot,
-

[1/3] incubator-ariatosca git commit: Storage is now sql based with SQLAlchemy based models [Forced Update!]

[mxmrlv]

Repository: incubator-ariatosca
Updated Branches:
  refs/heads/SQLAlchemy-based-models 7ba33f1d5 -> 2d6b9375d (forced update)


http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/2d6b9375/tests/orchestrator/workflows/api/test_task.py
--
diff --git a/tests/orchestrator/workflows/api/test_task.py 
b/tests/orchestrator/workflows/api/test_task.py
index 8536902..4da42c1 100644
--- a/tests/orchestrator/workflows/api/test_task.py
+++ b/tests/orchestrator/workflows/api/test_task.py
@@ -22,7 +22,7 @@ from aria.orchestrator.workflows import api
 from tests import mock
 
 
-@pytest.fixture()
+@pytest.fixture
 def ctx():
 """
 Create the following graph in storage:
@@ -30,50 +30,26 @@ def ctx():
 :return:
 """
 simple_context = mock.context.simple()
-dependency_node = mock.models.get_dependency_node()
-dependency_node_instance = mock.models.get_dependency_node_instance(
-dependency_node=dependency_node)
-
-relationship = mock.models.get_relationship(dependency_node)
-relationship_instance = mock.models.get_relationship_instance(
-relationship=relationship,
-target_instance=dependency_node_instance
-)
-
-dependent_node = mock.models.get_dependent_node(relationship)
-dependent_node_instance = mock.models.get_dependent_node_instance(
-dependent_node=dependent_node,
-relationship_instance=relationship_instance
-)
-
-simple_context.model.node.store(dependent_node)
-simple_context.model.node.store(dependency_node)
-simple_context.model.node_instance.store(dependent_node_instance)
-simple_context.model.node_instance.store(dependency_node_instance)
-simple_context.model.relationship.store(relationship)
-simple_context.model.relationship_instance.store(relationship_instance)
-simple_context.model.execution.store(mock.models.get_execution())
-simple_context.model.deployment.store(mock.models.get_deployment())
+
simple_context.model.execution.store(mock.models.get_execution(simple_context.deployment))
 
 return simple_context
 
 
 class TestOperationTask(object):
 
-def test_node_operation_task_creation(self):
-workflow_context = mock.context.simple()
-
+def test_node_operation_task_creation(self, ctx):
 operation_name = 'aria.interfaces.lifecycle.create'
 op_details = {'operation': True}
-node = mock.models.get_dependency_node()
+node = ctx.model.node.get(mock.models.DEPENDENT_NODE_ID)
 node.operations[operation_name] = op_details
-node_instance = 
mock.models.get_dependency_node_instance(dependency_node=node)
+ctx.model.node.update(node)
+node_instance = 
ctx.model.node_instance.get(mock.models.DEPENDENT_NODE_INSTANCE_ID)
 inputs = {'inputs': True}
 max_attempts = 10
 retry_interval = 10
 ignore_failure = True
 
-with context.workflow.current.push(workflow_context):
+with context.workflow.current.push(ctx):
 api_task = api.task.OperationTask.node_instance(
 name=operation_name,
 instance=node_instance,
@@ -90,19 +66,18 @@ class TestOperationTask(object):
 assert api_task.max_attempts == max_attempts
 assert api_task.ignore_failure == ignore_failure
 
-def test_relationship_operation_task_creation(self):
-workflow_context = mock.context.simple()
-
+def test_relationship_operation_task_creation(self, ctx):
 operation_name = 'aria.interfaces.relationship_lifecycle.preconfigure'
 op_details = {'operation': True}
-relationship = mock.models.get_relationship()
+relationship = ctx.model.relationship.get(mock.models.RELATIONSHIP_ID)
 relationship.source_operations[operation_name] = op_details
-relationship_instance = 
mock.models.get_relationship_instance(relationship=relationship)
+relationship_instance = ctx.model.relationship_instance.get(
+mock.models.RELATIONSHIP_INSTANCE_ID)
 inputs = {'inputs': True}
 max_attempts = 10
 retry_interval = 10
 
-with context.workflow.current.push(workflow_context):
+with context.workflow.current.push(ctx):
 api_task = api.task.OperationTask.relationship_instance(
 name=operation_name,
 instance=relationship_instance,
@@ -118,18 +93,19 @@ class TestOperationTask(object):
 assert api_task.retry_interval == retry_interval
 assert api_task.max_attempts == max_attempts
 
-def test_operation_task_default_values(self):
-workflow_context = mock.context.simple(task_ignore_failure=True)
-with context.workflow.current.push(workflow_context):
-model_task = api.task.OperationTask(
+def test_operation_task_default_values(self, ctx):
+dependency_node_instance = ctx.model.node_instance.get(
+

[2/3] incubator-ariatosca git commit: Storage is now sql based with SQLAlchemy based models

[mxmrlv]

http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/2d6b9375/aria/storage/mapi/sql.py
--
diff --git a/aria/storage/mapi/sql.py b/aria/storage/mapi/sql.py
new file mode 100644
index 000..f2ec0e5
--- /dev/null
+++ b/aria/storage/mapi/sql.py
@@ -0,0 +1,368 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+SQLalchemy based MAPI
+"""
+
+from sqlite3 import DatabaseError as SQLiteDBError
+from sqlalchemy.exc import SQLAlchemyError
+from sqlalchemy.sql.elements import Label
+
+from aria.utils.collections import OrderedDict
+
+
+try:
+from psycopg2 import DatabaseError as Psycopg2DBError
+sql_errors = (SQLAlchemyError, SQLiteDBError, Psycopg2DBError)
+except ImportError:
+sql_errors = (SQLAlchemyError, SQLiteDBError)
+Psycopg2DBError = None
+
+from ... import storage
+
+
+DEFAULT_SQL_DIALECT = 'sqlite'
+
+
+class SQLAlchemyModelAPI(storage.api.ModelAPI):
+"""
+SQL based MAPI.
+"""
+
+def __init__(self,
+ engine,
+ session,
+ **kwargs):
+super(SQLAlchemyModelAPI, self).__init__(**kwargs)
+self._engine = engine
+self._session = session
+
+def get(self, entry_id, include=None, filters=None, locking=False):
+"""Return a single result based on the model class and element ID
+"""
+filters = filters or {'id': entry_id}
+query = self._get_query(include, filters)
+if locking:
+query = query.with_for_update()
+result = query.first()
+
+if not result:
+raise storage.exceptions.StorageError(
+'Requested {0} with ID `{1}` was not found'
+.format(self.model_cls.__name__, entry_id)
+)
+return result
+
+def iter(self,
+ include=None,
+ filters=None,
+ pagination=None,
+ sort=None):
+"""Return a (possibly empty) list of `model_class` results
+"""
+query = self._get_query(include, filters, sort)
+
+results, _, _, _ = self._paginate(query, pagination)
+
+for result in results:
+yield result
+
+def store(self, entry, **kwargs):
+"""Create a `model_class` instance from a serializable `model` object
+
+:param entry: A dict with relevant kwargs, or an instance of a class
+that has a `to_dict` method, and whose attributes match the columns
+of `model_class` (might also my just an instance of `model_class`)
+:return: An instance of `model_class`
+"""
+self._session.add(entry)
+self._safe_commit()
+return entry
+
+def delete(self, entry_id, filters=None):
+"""Delete a single result based on the model class and element ID
+"""
+try:
+instance = self.get(
+entry_id,
+filters=filters
+)
+except storage.exceptions.StorageError:
+raise storage.exceptions.StorageError(
+'Could not delete {0} with ID `{1}` - element not found'
+.format(
+self.model_cls.__name__,
+entry_id
+)
+)
+self._load_properties(instance)
+self._session.delete(instance)
+self._safe_commit()
+return instance
+
+# TODO: this might need rework
+def update(self, entry, **kwargs):
+"""Add `instance` to the DB session, and attempt to commit
+
+:return: The updated instance
+"""
+return self.store(entry)
+
+def refresh(self, entry):
+"""Reload the instance with fresh information from the DB
+
+:param entry: Instance to be re-loaded from the DB
+:return: The refreshed instance
+"""
+self._session.refresh(entry)
+self._load_properties(entry)
+return entry
+
+def _destroy_connection(self):
+pass
+
+def _establish_connection(self):
+pass
+
+def create(self):
+self.model_cls.__table__.create(self._engine)
+
+def drop(self):
+"""
+Drop the table from the storage.
+ 

incubator-ariatosca git commit: lynting and tests fixing

[mxmrlv]

Repository: incubator-ariatosca
Updated Branches:
  refs/heads/SQLAlchemy-based-models 512b4bc6a -> 7ba33f1d5


lynting and tests fixing


Project: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/commit/7ba33f1d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/tree/7ba33f1d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/diff/7ba33f1d

Branch: refs/heads/SQLAlchemy-based-models
Commit: 7ba33f1d53ab0c482e0c733135a4d53bb18bb952
Parents: 512b4bc
Author: mxmrlv 
Authored: Thu Dec 1 12:31:25 2016 +0200
Committer: mxmrlv 
Committed: Thu Dec 1 12:31:25 2016 +0200

--
 aria/__init__.py| 30 
 aria/orchestrator/workflows/api/task.py |  9 +++--
 aria/storage/__init__.py| 38 +---
 aria/storage/api.py |  2 +-
 aria/storage/exceptions.py  |  4 ++-
 aria/storage/filesystem_api.py  | 13 ++-
 aria/storage/mapi/filesystem.py |  4 +--
 aria/storage/mapi/inmemory.py   |  2 +-
 aria/storage/mapi/sql.py| 13 ---
 aria/storage/models.py  |  2 +-
 aria/storage/rapi/filesystem.py | 18 ++
 aria/storage/structures.py  | 11 --
 .../orchestrator/workflows/builtin/__init__.py  |  5 ---
 tests/storage/test_resource_storage.py  |  2 +-
 14 files changed, 91 insertions(+), 62 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/7ba33f1d/aria/__init__.py
--
diff --git a/aria/__init__.py b/aria/__init__.py
index 1fd46c7..2c00729 100644
--- a/aria/__init__.py
+++ b/aria/__init__.py
@@ -63,21 +63,21 @@ def application_model_storage(api, api_params=None):
 Initiate model storage for the supplied storage driver
 """
 models = [
-storage.models.Blueprint,
-storage.models.Deployment,
-storage.models.Node,
-storage.models.NodeInstance,
-storage.models.Relationship,
-storage.models.RelationshipInstance,
-storage.models.Plugin,
-storage.models.Snapshot,
-storage.models.DeploymentUpdate,
-storage.models.DeploymentUpdateStep,
-storage.models.DeploymentModification,
-storage.models.Execution,
-storage.models.ProviderContext,
-storage.models.Task,
-]
+storage.models.Blueprint,
+storage.models.Deployment,
+storage.models.Node,
+storage.models.NodeInstance,
+storage.models.Relationship,
+storage.models.RelationshipInstance,
+storage.models.Plugin,
+storage.models.Snapshot,
+storage.models.DeploymentUpdate,
+storage.models.DeploymentUpdateStep,
+storage.models.DeploymentModification,
+storage.models.Execution,
+storage.models.ProviderContext,
+storage.models.Task,
+]
 # if api not in _model_storage:
 _model_storage[api] = storage.ModelStorage(api, items=models, 
api_params=api_params or {})
 return _model_storage[api]

http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/7ba33f1d/aria/orchestrator/workflows/api/task.py
--
diff --git a/aria/orchestrator/workflows/api/task.py 
b/aria/orchestrator/workflows/api/task.py
index e95ff6d..358315c 100644
--- a/aria/orchestrator/workflows/api/task.py
+++ b/aria/orchestrator/workflows/api/task.py
@@ -18,7 +18,6 @@ Provides the tasks to be entered into the task graph
 """
 from uuid import uuid4
 
-import aria
 from aria import storage
 
 from ... import context
@@ -76,8 +75,8 @@ class OperationTask(BaseTask):
 :param actor: the operation host on which this operation is registered.
 :param inputs: operation inputs.
 """
-assert isinstance(actor, (aria.storage.models.NodeInstance,
-  aria.storage.models.RelationshipInstance))
+assert isinstance(actor, (storage.models.NodeInstance,
+  storage.models.RelationshipInstance))
 super(OperationTask, self).__init__()
 self.actor = actor
 self.name = '{name}.{actor.id}'.format(name=name, actor=actor)
@@ -98,7 +97,7 @@ class OperationTask(BaseTask):
 :param instance: the node of which this operation belongs to.
 :param name: the name of the operation.
 """
-assert isinstance(instance,