On 02/06, lut...@redhat.com wrote: I really like the simplification you did. Altrough I will need to look deeper to understand how exactly the code works, I like it :-)
ACK. -- Michal > From: David Lutterkort <lut...@redhat.com> > > The 'entities' table now associates entries with CIMI::Model objects rather > than backend objects, and treats the DB as a proper look-aside cache. > > This has the advantage that converting from backend objects is decoupled > from looking up the common attributes we store in the database (name, > description, properties) > > The lookup of common attributes now happens when the model object is > constructed (at the point that the :id attribute is set, to be exact), and > can be saved with a call to its save method. > --- > server/lib/cimi/helpers/database_helper.rb | 62 +---------- > server/lib/cimi/models/address.rb | 16 +-- > server/lib/cimi/models/base.rb | 65 +++++++++++- > server/lib/cimi/models/machine.rb | 29 ++--- > server/lib/cimi/models/machine_image.rb | 20 ++-- > server/lib/cimi/models/volume.rb | 21 ++-- > server/lib/db/entity.rb | 35 ++++++ > server/lib/db/provider.rb | 10 ++ > server/tests/cimi/db/database_helper_test.rb | 153 > ++++++++++----------------- > 9 files changed, 209 insertions(+), 202 deletions(-) > > diff --git a/server/lib/cimi/helpers/database_helper.rb > b/server/lib/cimi/helpers/database_helper.rb > index 31459f4..c98b3b7 100644 > --- a/server/lib/cimi/helpers/database_helper.rb > +++ b/server/lib/cimi/helpers/database_helper.rb > @@ -18,25 +18,6 @@ module Deltacloud > return false > end > > - def load_attributes_for(model) > - return {} if test_environment? > - entity = get_entity(model) > - entity.nil? ? {} : entity.to_hash > - end > - > - def delete_attributes_for(model) > - return if test_environment? > - entity = get_entity(model) > - !entity.nil? && entity.destroy > - end > - > - def get_entity(model) > - current_db.entities_dataset.first( > - :be_kind => model.to_entity, > - :be_id => model.id > - ) > - end > - > def current_provider > Thread.current[:provider] || ENV['API_PROVIDER'] || 'default' > end > @@ -46,49 +27,8 @@ module Deltacloud > # > > def current_db > - Deltacloud::Database::Provider.find_or_create(:driver => > driver_symbol.to_s, :url => current_provider) > + Deltacloud::Database::Provider.lookup > end > - > - def store_attributes_for(model, attrs={}) > - return if test_environment? or model.nil? or attrs.empty? > - return if model.id.nil? > - > - unless entity = get_entity(model) > - entity = Deltacloud::Database::Entity.new( > - :provider_id => current_db.id, > - :be_id => model.id, > - :be_kind => model.to_entity > - ) > - end > - > - entity.description = extract_attribute_value('description', attrs) > if attrs.has_key? 'description' > - entity.name = extract_attribute_value('name', attrs) if > attrs.has_key? 'name' > - > - if attrs.has_key? 'properties' > - entity.ent_properties = extract_attribute_value('properties', > attrs).to_json > - elsif attrs.has_key? 'property' > - entity.ent_properties = extract_attribute_value('property', > attrs).to_json > - end > - > - entity.exists? ? entity.save_changes : entity.save > - > - entity > - end > - > - # In XML serialization the values stored in attrs are arrays, dues to > - # XmlSimple. This method will help extract values from them > - # > - def extract_attribute_value(name, attrs={}) > - return unless attrs[name] > - if name == 'property' > - attrs[name].is_a?(Array) ? > - attrs[name].inject({}) { |r, v| r[v['key']] = v['content']; r} : > - attrs[name] > - else > - attrs[name].is_a?(Array) ? attrs[name].first : attrs[name] > - end > - end > - > end > end > > diff --git a/server/lib/cimi/models/address.rb > b/server/lib/cimi/models/address.rb > index 5a1d34d..61d55dd 100644 > --- a/server/lib/cimi/models/address.rb > +++ b/server/lib/cimi/models/address.rb > @@ -65,23 +65,26 @@ class CIMI::Model::Address < CIMI::Model::Base > params = {:name=>input["name"], :description=>input["description"], > :address_template=>address_template, :env=>context } > raise CIMI::Model::BadRequest.new("Bad request - missing required > parameters. Client sent: #{request_body} which produced #{params.inspect}") > if params.has_value?(nil) > address = context.driver.create_address(context.credentials, params) > - store_attributes_for(address, input) > - from_address(address, context) > + result = from_address(address, context) > + result.name = input['name'] if input['name'] > + result.description = input['description'] if input['description'] > + result.extract_properties!(input) > + result.save > + result > end > > def self.delete!(id, context) > context.driver.delete_address(context.credentials, id) > - delete_attributes_for(::Address.new(:id => id)) > + new(:id => id).destroy > end > > private > > def self.from_address(address, context) > - stored_attributes = load_attributes_for(address) > self.new( > - :name => stored_attributes[:name] || address.id, > + :name => address.id, > :id => context.address_url(address.id), > - :description => stored_attributes[:description] || "Address > #{address.id}", > + :description => "Address #{address.id}", > :ip => address.id, > :allocation => "dynamic", #or "static" > :default_gateway => "unknown", #wtf > @@ -89,7 +92,6 @@ class CIMI::Model::Address < CIMI::Model::Base > :protocol => protocol_from_address(address.id), > :mask => "unknown", > :resource => (address.instance_id) ? {:href=> > context.machine_url(address.instance_id)} : nil, > - :property => stored_attributes[:property], > :network => nil #unknown > #optional: > #:hostname => > diff --git a/server/lib/cimi/models/base.rb b/server/lib/cimi/models/base.rb > index c74b8c3..ac838a4 100644 > --- a/server/lib/cimi/models/base.rb > +++ b/server/lib/cimi/models/base.rb > @@ -88,6 +88,69 @@ module CIMI::Model > # > text :id, :name, :description, :created > hash :property > - end > > + def initialize(values = {}) > + super(values) > + retrieve_entity > + end > + > + def []=(a, v) > + v = super > + retrieve_entity if a == :id > + v > + end > + > + # Save the common attributes name, description, and properties to the > + # database > + def save > + if @entity > + @entity.name = name > + @entity.description = description > + @entity.properties = property > + @entity.save > + end > + self > + end > + > + # Destroy the database attributes for this model > + def destroy > + @entity.destroy > + self > + end > + > + # FIXME: Kludge around the fact that we do not have proper *Create > + # objects that deserialize properties by themselves > + def extract_properties!(data) > + h = {} > + if data['property'] > + # Data came from XML > + h = data['property'].inject({}) do |r,v| > + r[v['key']] = v['content'] > + r > + end > + elsif data['properties'] > + h = data['properties'] > + end > + property ||= {} > + property.merge!(h) > + end > + > + private > + > + # Load an existing database entity for this object, or create a new one > + def retrieve_entity > + if self.id > + @entity = Deltacloud::Database::Entity::retrieve(self) > + if @entity.exists? > + self.name = @entity.name > + self.description = @entity.description > + self.property ||= {} > + self.property.merge!(@entity.properties) > + end > + else > + @entity = nil > + end > + end > + > + end > end > diff --git a/server/lib/cimi/models/machine.rb > b/server/lib/cimi/models/machine.rb > index 214af42..a44332a 100644 > --- a/server/lib/cimi/models/machine.rb > +++ b/server/lib/cimi/models/machine.rb > @@ -76,8 +76,12 @@ class CIMI::Model::Machine < CIMI::Model::Base > > # Store attributes that are not supported by the backend cloud to local > # database: > - store_attributes_for(instance, json) > - from_instance(instance, context) > + machine = from_instance(instance, context) > + machine.name = json['name'] || machine.name > + machine.description = json['description'] > + machine.extract_properties!(json) > + machine.save > + machine > end > > def self.create_from_xml(body, context) > @@ -105,8 +109,12 @@ class CIMI::Model::Machine < CIMI::Model::Base > > # Store attributes that are not supported by the backend cloud to local > # database: > - entity = store_attributes_for(instance, xml) > - from_instance(instance, context, entity.to_hash) > + machine = from_instance(instance, context) > + machine.name = xml['name'] || machine.name > + machine.description = xml['description'] > + machine.extract_properties!(xml) > + machine.save > + machine > end > > def perform(action, context, &block) > @@ -122,8 +130,8 @@ class CIMI::Model::Machine < CIMI::Model::Base > end > > def self.delete!(id, context) > - delete_attributes_for Instance.new(:id => id) > context.driver.destroy_instance(context.credentials, id) > + CIMI::Model::Machine.new(:id => id).delete > end > > #returns the newly attach machine_volume > @@ -141,10 +149,9 @@ class CIMI::Model::Machine < CIMI::Model::Base > end > > private > - def self.from_instance(instance, context, stored_attributes=nil) > + def self.from_instance(instance, context) > cpu = memory = (instance.instance_profile.id == "opaque")? "n/a" : nil > machine_conf = > CIMI::Model::MachineConfiguration.find(instance.instance_profile.name, > context) > - stored_attributes ||= load_attributes_for(instance) > machine_spec = { > :name => instance.name, > :created => instance.launch_time.nil? ? Time.now.xmlschema : > Time.parse(instance.launch_time.to_s).xmlschema, > @@ -155,9 +162,8 @@ class CIMI::Model::Machine < CIMI::Model::Base > :memory => memory || > convert_instance_memory(instance.instance_profile, context), > :disks => { :href => context.machine_url(instance.id)+"/disks"}, > :volumes => { :href=>context.machine_url(instance.id)+"/volumes"}, > - :operations => convert_instance_actions(instance, context), > - :property => stored_attributes > - }.merge(stored_attributes) > + :operations => convert_instance_actions(instance, context) > + } > if context.expand? :disks > machine_spec[:disks] = CIMI::Model::Disk.find(instance, machine_conf, > context, :all) > end > @@ -166,8 +172,7 @@ class CIMI::Model::Machine < CIMI::Model::Base > end > machine_spec[:realm] = instance.realm_id if instance.realm_id > machine_spec[:machine_image] = { :href => > context.machine_image_url(instance.image_id) } if instance.image_id > - machine = self.new(machine_spec) > - machine > + self.new(machine_spec) > end > > # FIXME: This will convert 'RUNNING' state to 'STARTED' > diff --git a/server/lib/cimi/models/machine_image.rb > b/server/lib/cimi/models/machine_image.rb > index b521b49..247ff72 100644 > --- a/server/lib/cimi/models/machine_image.rb > +++ b/server/lib/cimi/models/machine_image.rb > @@ -38,17 +38,13 @@ class CIMI::Model::MachineImage < CIMI::Model::Base > end > > def self.from_image(image, context) > - stored_attributes = load_attributes_for(image) > self.new( > :id => context.machine_image_url(image.id), > - :name => stored_attributes[:name] || image.id, > - :description => stored_attributes[:description] || image.description, > + :name => image.id, > + :description => image.description, > :state => image.state || 'UNKNOWN', > :type => "IMAGE", > - :created => image.creation_time.nil? ? Time.now.xmlschema : > Time.parse(image.creation_time.to_s).xmlschema, > - :image_location => (stored_attributes[:property] && > stored_attributes[:property]['image_location']) ? > - stored_attributes[:property].delete('image_location') : 'unknown', > - :property => stored_attributes[:property] > + :created => image.creation_time.nil? ? Time.now.xmlschema : > Time.parse(image.creation_time.to_s).xmlschema > ) > end > > @@ -71,13 +67,17 @@ class CIMI::Model::MachineImage < CIMI::Model::Base > end > params = {:id => context.href_id(input["imageLocation"], :machines), > :name=>input["name"], :description=>input["description"]} > image = context.driver.create_image(context.credentials, params) > - store_attributes_for(image, input) > - from_image(image, context) > + result = from_image(image, context) > + result.name = input['name'] if input['name'] > + result.description = input['description'] if input['description'] > + result.extract_properties!(input) > + result.save > + result > end > > def self.delete!(image_id, context) > context.driver.destroy_image(context.credentials, image_id) > - delete_attributes_for(::Image.new(:id => image_id)) > + CIMI::Model::Image.new(:id => image_id).delete > end > > end > diff --git a/server/lib/cimi/models/volume.rb > b/server/lib/cimi/models/volume.rb > index b4503fc..3bb0f10 100644 > --- a/server/lib/cimi/models/volume.rb > +++ b/server/lib/cimi/models/volume.rb > @@ -71,7 +71,7 @@ class CIMI::Model::Volume < CIMI::Model::Base > > def self.delete!(id, context) > context.driver.destroy_storage_volume(context.credentials, {:id=>id} ) > - delete_attributes_for(StorageVolume.new(:id => id)) > + new(:id => id).destroy > end > > def self.find_to_attach_from_json(json_in, context) > @@ -86,10 +86,6 @@ class CIMI::Model::Volume < CIMI::Model::Base > :attachment_point=>v["attachmentPoint"] }} > end > > - def to_entity > - 'volume' > - end > - > private > > def self.create_volume(params, data, context) > @@ -104,15 +100,16 @@ class CIMI::Model::Volume < CIMI::Model::Base > :name=>data["name"]} > end > storage_volume = > context.driver.create_storage_volume(context.credentials, opts) > - entity = store_attributes_for(storage_volume, data) > - from_storage_volume(storage_volume, context, entity.to_hash) > + result = from_storage_volume(storage_volume, context) > + result.name = data['name'] if data['name'] > + result.description = data['description'] > + result.extract_properties!(data) > + result.save > + result > end > > - def self.from_storage_volume(volume, context, stored_attributes=nil) > - stored_attributes ||= load_attributes_for(volume) > - self.new( { :name => stored_attributes[:name] || volume.id, > - :description => stored_attributes[:description] || > 'Description of Volume', > - :property => stored_attributes[:property], > + def self.from_storage_volume(volume, context) > + self.new( { :name => volume.id, > :created => volume.created.nil? ? nil : > Time.parse(volume.created).xmlschema, > :id => context.volume_url(volume.id), > :capacity => context.to_kibibyte(volume.capacity, 'GB'), > diff --git a/server/lib/db/entity.rb b/server/lib/db/entity.rb > index 963a26b..08ed70b 100644 > --- a/server/lib/db/entity.rb > +++ b/server/lib/db/entity.rb > @@ -3,6 +3,8 @@ module Deltacloud > > class Entity < Sequel::Model > > + attr_accessor :properties > + > many_to_one :provider > > plugin :single_table_inheritance, :model > @@ -16,6 +18,39 @@ module Deltacloud > retval > end > > + def before_save > + self.ent_properties = properties.to_json > + super > + end > + > + def after_initialize > + if ent_properties > + self.properties = JSON::parse(ent_properties) > + else > + self.properties = {} > + end > + end > + > + # Load the entity for the CIMI::Model +model+, or create a new one if > + # none exists yet > + def self.retrieve(model) > + unless model.id > + raise "Can not retrieve entity for #{model.class.name} without an > id" > + end > + h = model_hash(model) > + entity = Provider::lookup.entities_dataset.first(h) > + unless entity > + h[:provider_id] = Provider::lookup.id > + entity = Entity.new(h) > + end > + entity > + end > + > + private > + def self.model_hash(model) > + { :be_kind => model.class.name, > + :be_id => model.id.split("/").last } > + end > end > > end > diff --git a/server/lib/db/provider.rb b/server/lib/db/provider.rb > index c0b8485..ee3e138 100644 > --- a/server/lib/db/provider.rb > +++ b/server/lib/db/provider.rb > @@ -1,12 +1,22 @@ > +require_relative '../deltacloud/helpers/driver_helper' > + > module Deltacloud > module Database > > class Provider < Sequel::Model > + extend Deltacloud::Helpers::Drivers > + > one_to_many :entities > one_to_many :machine_templates > one_to_many :address_templates > one_to_many :volume_templates > one_to_many :volume_configurations > + > + # Find the DB provider set in the environment/request > + def self.lookup > + prov = Thread.current[:provider] || ENV['API_PROVIDER'] || 'default' > + find_or_create(:driver => self.driver_symbol.to_s, :url => prov) > + end > end > > end > diff --git a/server/tests/cimi/db/database_helper_test.rb > b/server/tests/cimi/db/database_helper_test.rb > index 7bd3e23..30772f8 100644 > --- a/server/tests/cimi/db/database_helper_test.rb > +++ b/server/tests/cimi/db/database_helper_test.rb > @@ -13,9 +13,14 @@ end > describe Deltacloud::Helpers::Database do > include Deltacloud::DatabaseTestHelper > > + Provider = Deltacloud::Database::Provider > + Entity = Deltacloud::Database::Entity > + BaseModel = CIMI::Model::Base > + > before do > ENV['RACK_ENV'] = 'development' > @db = DatabaseHelper.new > + @prov = Provider::lookup > end > > it 'report if application is running under test environment' do > @@ -35,97 +40,84 @@ describe Deltacloud::Helpers::Database do > end > > it 'create provider when it does not exists' do > - @db.current_db.must_be_kind_of Deltacloud::Database::Provider > - @db.current_db.driver.must_equal 'mock' > - @db.current_db.url.must_equal @db.current_provider > - @db.current_db.must_respond_to :entities > - @db.current_db.must_respond_to :machine_templates > - @db.current_db.must_respond_to :address_templates > - @db.current_db.must_respond_to :volume_configurations > - @db.current_db.must_respond_to :volume_templates > - end > - > - it 'extract attributes both from JSON and XMLSimple' do > - xml_simple_test = { 'property' => [ { 'key' => 'template', 'content' => > "value"} ] } > - json_test = { 'properties' => { 'template' => 'value' } } > - > - @db.extract_attribute_value('property', > xml_simple_test).must_equal('template' => 'value') > - @db.extract_attribute_value('properties', > json_test).must_equal('template' => 'value') > + @prov.must_be_kind_of Deltacloud::Database::Provider > + @prov.driver.must_equal 'mock' > + @prov.url.must_equal @db.current_provider > + @prov.must_respond_to :entities > + @prov.must_respond_to :machine_templates > + @prov.must_respond_to :address_templates > + @prov.must_respond_to :volume_configurations > + @prov.must_respond_to :volume_templates > end > > it 'must return entity for given model' do > - provider = Deltacloud::Database::Provider > - entity = Deltacloud::Database::Entity > - @db.current_db.wont_be_nil > + @prov.wont_be_nil > > - new_entity = @db.current_db.add_entity( > + new_entity = @prov.add_entity( > :name => 'testMachine1', > :description => 'testMachine1 description', > :ent_properties => JSON::dump(:key => 'value'), > - :be_kind => 'instance', > + :be_kind => BaseModel.name, > :be_id => 'inst1' > ) > > - check_entity_base_attrs new_entity, entity, @db.current_db > + check_entity_base_attrs new_entity, Entity, @prov > > - result = @db.get_entity(Instance.new(:id => 'inst1')) > + result = Entity.retrieve(BaseModel.new(:id => 'inst1')) > result.must_equal new_entity > > - new_entity.destroy.wont_equal false > + new_entity.destroy > + result = Entity.retrieve(BaseModel.new(:id => 'inst1')) > + result.exists?.must_equal false > end > > it 'must load attributes for entity for given model' do > - provider = Deltacloud::Database::Provider > - entity = Deltacloud::Database::Entity > - @db.current_db.wont_be_nil > + @prov.wont_be_nil > > - new_entity = @db.current_db.add_entity( > + new_entity = @prov.add_entity( > :name => 'testMachine1', > :description => 'testMachine1 description', > :ent_properties => JSON::dump(:key => 'value'), > - :be_kind => 'instance', > - :be_id => 'inst1' > + :be_kind => BaseModel.name, > + :be_id => 'base1' > ) > > - check_entity_base_attrs new_entity, entity, @db.current_db > + check_entity_base_attrs new_entity, Entity, @prov > > - result = @db.load_attributes_for(Instance.new(:id => 'inst1')) > - result.must_be_kind_of Hash > - result[:name].must_equal new_entity.name > - result[:description].must_equal new_entity.description > - result[:property].must_equal JSON::parse(new_entity.ent_properties) > + result = Entity::retrieve(BaseModel.new(:id => 'base1')) > + result.name.must_equal new_entity.name > + result.description.must_equal new_entity.description > + result.properties.must_equal new_entity.properties > > new_entity.destroy.wont_equal false > end > > it 'must delete attributes for entity for given model' do > - provider = Deltacloud::Database::Provider > - entity = Deltacloud::Database::Entity > - @db.current_db.wont_be_nil > + @prov.wont_be_nil > > - new_entity = @db.current_db.add_entity( > + new_entity = @prov.add_entity( > :name => 'testMachine1', > :description => 'testMachine1 description', > :ent_properties => JSON::dump(:key => 'value'), > - :be_kind => 'instance', > - :be_id => 'inst1' > + :be_kind => BaseModel.name, > + :be_id => 'base1' > ) > > - check_entity_base_attrs new_entity, entity, @db.current_db > + check_entity_base_attrs new_entity, Entity, @prov > > - result = @db.delete_attributes_for(Instance.new(:id => 'inst1')) > - result.wont_equal false > - result.exists?.must_equal false > + base = BaseModel.new(:id => 'base1') > + base.destroy > + entity = Entity.retrieve(base) > + entity.wont_be_nil > + entity.exists?.must_equal false > end > > - it 'must store JSON attributes for entity for given model' do > - provider = Deltacloud::Database::Provider > - entity = Deltacloud::Database::Entity > - @db.current_db.wont_be_nil > + it 'must store attributes for a given CIMI::Model' do > + @prov.wont_be_nil > > - mock_instance = Instance.new(:id => 'inst1') > - mock_json = ' > + json = ' > { > + "id": "http://localhost:3001/cimi/machines/42", > "resourceURI": "http://schemas.dmtf.org/cimi/1/MachineCreate", > "name": "myDatabaseMachine", > "description": "This is a demo machine", > @@ -139,52 +131,15 @@ describe Deltacloud::Helpers::Database do > } > } > ' > - result = @db.store_attributes_for(mock_instance, JSON::parse(mock_json)) > - result.must_be_kind_of entity > - check_entity_base_attrs result, entity, @db.current_db > - load_result = @db.load_attributes_for(mock_instance) > - load_result.must_be_kind_of Hash > - load_result.wont_be_empty > - load_result[:name].must_equal 'myDatabaseMachine' > - load_result[:description].must_equal 'This is a demo machine' > - load_result[:property].must_be_kind_of Hash > - load_result[:property].wont_be_empty > - load_result[:property]['foo'].must_equal 'bar' > - load_result[:property]['life'].must_equal 'is life' > - result.destroy.wont_equal false > + machine = CIMI::Model::Machine.from_json(json) > + machine.save > + > + m2 = CIMI::Model::Machine.new(:id => machine.id) > + m2.name.must_equal 'myDatabaseMachine' > + m2.description.must_equal 'This is a demo machine' > + m2.property.must_be_kind_of Hash > + m2.property.size.must_equal 2 > + m2.property['foo'].must_equal 'bar' > + m2.property['life'].must_equal 'is life' > end > - > - it 'must store XML attributes for entity for given model' do > - provider = Deltacloud::Database::Provider > - entity = Deltacloud::Database::Entity > - @db.current_db.wont_be_nil > - > - mock_instance = Instance.new(:id => 'inst1') > - mock_xml = ' > -<MachineCreate> > - <name>myMachineXML123</name> > - <description>Description of my new Machine</description> > - <machineTemplate> > - <machineConfig > href="http://localhost:3001/cimi/machine_configurations/m1-small"/> > - <machineImage href="http://localhost:3001/cimi/machine_images/img1"/> > - </machineTemplate> > - <property key="test">value</property> > - <property key="foo">bar</property> > -</MachineCreate> > - ' > - result = @db.store_attributes_for(mock_instance, > XmlSimple.xml_in(mock_xml)) > - result.must_be_kind_of entity > - check_entity_base_attrs result, entity, @db.current_db > - load_result = @db.load_attributes_for(mock_instance) > - load_result.must_be_kind_of Hash > - load_result.wont_be_empty > - load_result[:name].must_equal 'myMachineXML123' > - load_result[:description].must_equal 'Description of my new Machine' > - load_result[:property].must_be_kind_of Hash > - load_result[:property].wont_be_empty > - load_result[:property]['test'].must_equal 'value' > - result.destroy.wont_equal false > - end > - > - > end > -- > 1.8.1 > -- Michal Fojtik <mfoj...@redhat.com> Deltacloud API, CloudForms