Repository: incubator-usergrid Updated Branches: refs/heads/entity-size [created] 5e56f1b9c
add entity size Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/5e56f1b9 Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/5e56f1b9 Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/5e56f1b9 Branch: refs/heads/entity-size Commit: 5e56f1b9cdec740c9907350ed1b585f09f06752a Parents: 72016ea Author: Shawn Feldman <sfeld...@apache.org> Authored: Thu Aug 20 12:59:22 2015 -0700 Committer: Shawn Feldman <sfeld...@apache.org> Committed: Thu Aug 20 12:59:22 2015 -0700 ---------------------------------------------------------------------- .../corepersistence/CpEntityManager.java | 5 ++-- .../usergrid/persistence/AbstractEntity.java | 7 ++++++ .../org/apache/usergrid/persistence/Entity.java | 3 +++ .../persistence/collection/MvccEntity.java | 6 +++++ .../mvcc/entity/impl/MvccEntityImpl.java | 25 +++++++++++++++++--- .../collection/mvcc/stage/delete/MarkStart.java | 2 +- .../MvccEntitySerializationStrategyImpl.java | 4 ++-- .../MvccEntitySerializationStrategyV3Impl.java | 24 +++++++++++++------ ...ccEntitySerializationStrategyV3ImplTest.java | 1 - .../persistence/model/entity/Entity.java | 9 +++++++ .../index/impl/EntityToMapConverter.java | 5 +++- .../persistence/index/impl/IndexingUtils.java | 2 ++ 12 files changed, 76 insertions(+), 17 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5e56f1b9/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java ---------------------------------------------------------------------- diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java index 6611b6f..92464f8 100644 --- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java +++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java @@ -2615,7 +2615,8 @@ public class CpEntityManager implements EntityManager { } ); } - cpEntity = ecm .write( cpEntity ).toBlocking().last(); + cpEntity = ecm.write( cpEntity ).toBlocking().last(); + entity.setSize(cpEntity.getSize()); if(logger.isDebugEnabled()) { logger.debug( "Wrote {}:{} version {}", new Object[] { @@ -2647,7 +2648,7 @@ public class CpEntityManager implements EntityManager { String collectionName = Schema.defaultCollectionName( eType ); CpRelationManager cpr = ( CpRelationManager ) getRelationManager( getApplication() ); - cpr.addToCollection( collectionName, entity); + cpr.addToCollection( collectionName, entity ); // Invoke counters incrementEntityCollection( collectionName, timestamp ); http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5e56f1b9/stack/core/src/main/java/org/apache/usergrid/persistence/AbstractEntity.java ---------------------------------------------------------------------- diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/AbstractEntity.java b/stack/core/src/main/java/org/apache/usergrid/persistence/AbstractEntity.java index afe246d..59106ba 100644 --- a/stack/core/src/main/java/org/apache/usergrid/persistence/AbstractEntity.java +++ b/stack/core/src/main/java/org/apache/usergrid/persistence/AbstractEntity.java @@ -62,6 +62,7 @@ public abstract class AbstractEntity implements Entity { protected Map<String, Object> dynamic_properties = new TreeMap<String, Object>( String.CASE_INSENSITIVE_ORDER ); protected Map<String, Set<Object>> dynamic_sets = new TreeMap<String, Set<Object>>( String.CASE_INSENSITIVE_ORDER ); + protected long size; @Override @@ -96,7 +97,12 @@ public abstract class AbstractEntity implements Entity { return new SimpleId( uuid, getType() ); } + @Override + public void setSize(final long size){this.size = size;} + + @Override + public long getSize(){return size;} @Override @EntityProperty(indexed = true, required = true, mutable = false) @@ -363,4 +369,5 @@ public abstract class AbstractEntity implements Entity { } return true; } + } http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5e56f1b9/stack/core/src/main/java/org/apache/usergrid/persistence/Entity.java ---------------------------------------------------------------------- diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/Entity.java b/stack/core/src/main/java/org/apache/usergrid/persistence/Entity.java index 2030bd1..b6176b8 100644 --- a/stack/core/src/main/java/org/apache/usergrid/persistence/Entity.java +++ b/stack/core/src/main/java/org/apache/usergrid/persistence/Entity.java @@ -102,4 +102,7 @@ public interface Entity extends EntityRef, Comparable<Entity> { @JsonAnyGetter public abstract Map<String, Object> getDynamicProperties(); + + long getSize(); + void setSize(long size); } http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5e56f1b9/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/MvccEntity.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/MvccEntity.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/MvccEntity.java index c89838e..53142c8 100644 --- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/MvccEntity.java +++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/MvccEntity.java @@ -68,4 +68,10 @@ public interface MvccEntity extends EntityVersion{ * Get the status of the entity */ Status getStatus(); + + /** + * entity byte size + * @return + */ + long getSize(); } http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5e56f1b9/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/entity/impl/MvccEntityImpl.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/entity/impl/MvccEntityImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/entity/impl/MvccEntityImpl.java index 14889bd..e0066ba 100644 --- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/entity/impl/MvccEntityImpl.java +++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/entity/impl/MvccEntityImpl.java @@ -38,24 +38,34 @@ public class MvccEntityImpl implements MvccEntity { private final UUID version; private final Optional<Entity> entity; private final Status status; + private final long size; public MvccEntityImpl( final Id entityId, final UUID version, final Status status, final Entity entity ) { - this( entityId, version, status, Optional.of( entity ) ); + this( entityId, version, status, entity, 0 ); } - + public MvccEntityImpl( final Id entityId, final UUID version, final Status status, final Entity entity, final long size ) { + this( entityId, version, status, Optional.of( entity ), size); + } public MvccEntityImpl( - final Id entityId, final UUID version, final Status status, final Optional<Entity> entity ) { + final Id entityId, final UUID version, final Status status, final Optional<Entity> entity ) { + this( entityId, version, status, entity , 0); + } + + public MvccEntityImpl( + final Id entityId, final UUID version, final Status status, final Optional<Entity> entity, final long size ) { Preconditions.checkNotNull( entityId, "entity id is required" ); Preconditions.checkNotNull( version, "version id is required" ); Preconditions.checkNotNull( status, "status is required" ); Preconditions.checkNotNull( entity, "entity is required" ); + Preconditions.checkNotNull( size, "size is required" ); this.entityId = entityId; this.version = version; this.entity = entity; this.status = status; + this.size = size; } @@ -82,6 +92,11 @@ public class MvccEntityImpl implements MvccEntity { return status; } + @Override + public long getSize() { + return size; + } + @Override public boolean equals( final Object o ) { @@ -104,6 +119,10 @@ public class MvccEntityImpl implements MvccEntity { return false; } + if( size != that.size){ + return false; + } + return true; } http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5e56f1b9/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/delete/MarkStart.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/delete/MarkStart.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/delete/MarkStart.java index 040e893..3c4ea95 100644 --- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/delete/MarkStart.java +++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/delete/MarkStart.java @@ -106,7 +106,7 @@ public class MarkStart implements Func1<CollectionIoEvent<Id>, CollectionIoEvent //create the mvcc entity for the next stage final MvccEntityImpl nextStage = new MvccEntityImpl( - entityId, version, MvccEntity.Status.COMPLETE, Optional.<Entity>absent() ); + entityId, version, MvccEntity.Status.COMPLETE, Optional.<Entity>absent(), 0 ); return new CollectionIoEvent<MvccEntity>( applicationScope, nextStage ); http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5e56f1b9/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImpl.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImpl.java index 6fa539a..457664f 100644 --- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImpl.java +++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImpl.java @@ -424,7 +424,7 @@ public abstract class MvccEntitySerializationStrategyImpl implements MvccEntityS + " write was truncated.", id, version, e ); //return an empty entity, we can never load this one, and we don't want it to bring the system //to a grinding halt - return new MvccEntityImpl( id, version, MvccEntity.Status.DELETED, Optional.<Entity>absent() ); + return new MvccEntityImpl( id, version, MvccEntity.Status.DELETED, Optional.<Entity>absent(),0 ); } //Inject the id into it. @@ -432,7 +432,7 @@ public abstract class MvccEntitySerializationStrategyImpl implements MvccEntityS EntityUtils.setId( deSerialized.entity.get(), id ); } - return new MvccEntityImpl( id, version, deSerialized.status, deSerialized.entity ); + return new MvccEntityImpl( id, version, deSerialized.status, deSerialized.entity, 0 ); } } http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5e56f1b9/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV3Impl.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV3Impl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV3Impl.java index 81a0e6d..2f4c625 100644 --- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV3Impl.java +++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV3Impl.java @@ -112,7 +112,7 @@ public class MvccEntitySerializationStrategyV3Impl implements MvccEntitySerializ Optional<EntityMap> map = EntityMap.fromEntity(entity.getEntity()); return doWrite( applicationScope, entityId, version, colMutation -> colMutation.putColumn( COL_VALUE, - entitySerializer.toByteBuffer( new EntityWrapper(entityId,entity.getVersion(), entity.getStatus(), map.isPresent() ? map.get() : null ) ) ) ); + entitySerializer.toByteBuffer( new EntityWrapper(entityId,entity.getVersion(), entity.getStatus(), map.isPresent() ? map.get() : null, 0 ) ) ) ); } @@ -268,7 +268,7 @@ public class MvccEntitySerializationStrategyV3Impl implements MvccEntitySerializ return doWrite(applicationScope, entityId, version, colMutation -> colMutation.putColumn(COL_VALUE, - entitySerializer.toByteBuffer(new EntityWrapper(entityId, version, MvccEntity.Status.DELETED, null)) + entitySerializer.toByteBuffer(new EntityWrapper(entityId, version, MvccEntity.Status.DELETED, null, 0)) ) ); } @@ -355,10 +355,10 @@ public class MvccEntitySerializationStrategyV3Impl implements MvccEntitySerializ //return an empty entity, we can never load this one, and we don't want it to bring the system //to a grinding halt //TODO fix this - return new MvccEntityImpl( id, UUIDGenerator.newTimeUUID(), MvccEntity.Status.DELETED, Optional.<Entity>absent() ); + return new MvccEntityImpl( id, UUIDGenerator.newTimeUUID(), MvccEntity.Status.DELETED, Optional.<Entity>absent(),0 ); } Optional<Entity> entity = deSerialized.getOptionalEntity() ; - return new MvccEntityImpl( id, deSerialized.getVersion(), deSerialized.getStatus(), entity ); + return new MvccEntityImpl( id, deSerialized.getVersion(), deSerialized.getStatus(), entity, deSerialized.getSize()); } } @@ -453,6 +453,7 @@ public class MvccEntitySerializationStrategyV3Impl implements MvccEntitySerializ byte[] arr = byteBuffer.array(); bytesOutHistorgram.update( arr == null ? 0 : arr.length); entityWrapper = MAPPER.readValue(arr, EntityWrapper.class); + entityWrapper.size = arr.length; time.stop(); } catch ( Exception e ) { @@ -464,7 +465,7 @@ public class MvccEntitySerializationStrategyV3Impl implements MvccEntitySerializ // it's been deleted, remove it if ( entityWrapper.getEntityMap() == null) { - return new EntityWrapper( entityWrapper.getId(), entityWrapper.getVersion(),MvccEntity.Status.DELETED,null ); + return new EntityWrapper( entityWrapper.getId(), entityWrapper.getVersion(),MvccEntity.Status.DELETED,null,0 ); } entityWrapper.setStatus(MvccEntity.Status.COMPLETE); @@ -482,15 +483,17 @@ public class MvccEntitySerializationStrategyV3Impl implements MvccEntitySerializ private MvccEntity.Status status; private UUID version; private EntityMap entityMap; + private long size; public EntityWrapper( ) { } - public EntityWrapper( final Id id , final UUID version, final MvccEntity.Status status, final EntityMap entity ) { + public EntityWrapper( final Id id , final UUID version, final MvccEntity.Status status, final EntityMap entity, final long size ) { this.setStatus(status); this.version= version; this.entityMap = entity; this.id = id; + this.size = size; } /** @@ -525,7 +528,9 @@ public class MvccEntitySerializationStrategyV3Impl implements MvccEntitySerializ @JsonIgnore public Optional<Entity> getOptionalEntity() { - Optional<Entity> entityReturn = Optional.fromNullable(Entity.fromMap(getEntityMap())); + Entity entity = Entity.fromMap(getEntityMap()); + entity.setSize(getSize()); + Optional<Entity> entityReturn = Optional.fromNullable(entity); //Inject the id into it. if (entityReturn.isPresent()) { EntityUtils.setId(entityReturn.get(), getId()); @@ -534,6 +539,11 @@ public class MvccEntitySerializationStrategyV3Impl implements MvccEntitySerializ ; return entityReturn; } + + @JsonIgnore + public long getSize() { + return size; + } } /** http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5e56f1b9/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV3ImplTest.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV3ImplTest.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV3ImplTest.java index 1df22f6..d55894b 100644 --- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV3ImplTest.java +++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV3ImplTest.java @@ -54,7 +54,6 @@ public class MvccEntitySerializationStrategyV3ImplTest extends MvccEntitySeriali @Test( expected = UnsupportedOperationException.class ) public void loadDescendingHistory() throws ConnectionException { - final String name = "test"; final Id applicationId = new SimpleId( "application" ); http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5e56f1b9/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/entity/Entity.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/entity/Entity.java b/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/entity/Entity.java index cd75544..013b37a 100644 --- a/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/entity/Entity.java +++ b/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/entity/Entity.java @@ -59,6 +59,9 @@ public class Entity extends EntityObject { @JsonProperty private UUID version; + @JsonIgnore + private long size; + /** * Create an entity with the given type and id. Should be used for all update operations to an existing entity @@ -155,4 +158,10 @@ public class Entity extends EntityObject { return getVersion() != null; } + public long getSize() { + return this.size; + } + public void setSize(long size) { + this.size = size; + } } http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5e56f1b9/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EntityToMapConverter.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EntityToMapConverter.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EntityToMapConverter.java index 567068c..1b0d110 100644 --- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EntityToMapConverter.java +++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EntityToMapConverter.java @@ -35,6 +35,7 @@ import static org.apache.usergrid.persistence.index.impl.IndexingUtils.EDGE_SEAR import static org.apache.usergrid.persistence.index.impl.IndexingUtils.EDGE_TIMESTAMP_FIELDNAME; import static org.apache.usergrid.persistence.index.impl.IndexingUtils.ENTITY_FIELDS; import static org.apache.usergrid.persistence.index.impl.IndexingUtils.ENTITY_ID_FIELDNAME; +import static org.apache.usergrid.persistence.index.impl.IndexingUtils.ENTITY_SIZE_FIELDNAME; import static org.apache.usergrid.persistence.index.impl.IndexingUtils.ENTITY_TYPE_FIELDNAME; import static org.apache.usergrid.persistence.index.impl.IndexingUtils.ENTITY_VERSION_FIELDNAME; import static org.apache.usergrid.persistence.index.impl.IndexingUtils.applicationId; @@ -71,7 +72,7 @@ public class EntityToMapConverter { outputEntity.put( ENTITY_VERSION_FIELDNAME, entity.getVersion() ); - outputEntity.put( ENTITY_TYPE_FIELDNAME, getType( applicationScope, entityId ) ); + outputEntity.put( ENTITY_TYPE_FIELDNAME, getType( applicationScope, entityId)); outputEntity.put( APPLICATION_ID_FIELDNAME, applicationId( applicationScope.getApplication() ) ); @@ -83,6 +84,8 @@ public class EntityToMapConverter { outputEntity.put( EDGE_TIMESTAMP_FIELDNAME, indexEdge.getTimestamp() ); + outputEntity.put( ENTITY_SIZE_FIELDNAME, entity.getSize() ); + //add the context for filtering later outputEntity.put( EDGE_SEARCH_FIELDNAME, IndexingUtils.createContextName( applicationScope, indexEdge ) ); http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5e56f1b9/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java index e82a082..9e06fa6 100644 --- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java +++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java @@ -84,6 +84,8 @@ public class IndexingUtils { public static final String ENTITY_ID_FIELDNAME = "entityId"; + public static final String ENTITY_SIZE_FIELDNAME = "entitySize"; + public static final String ENTITY_VERSION_FIELDNAME = "entityVersion"; public static final String ENTITY_TYPE_FIELDNAME = "entityType";
