Finished changes to reflect new single index health.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/abbdd6b4 Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/abbdd6b4 Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/abbdd6b4 Branch: refs/heads/USERGRID-509 Commit: abbdd6b4869211a03dd14319e5fc5f57f71320a5 Parents: 6bf2d3d Author: Todd Nine <[email protected]> Authored: Fri Mar 27 09:41:17 2015 -0600 Committer: Todd Nine <[email protected]> Committed: Fri Mar 27 09:41:17 2015 -0600 ---------------------------------------------------------------------- .../corepersistence/CpEntityManager.java | 150 +++++++------------ .../corepersistence/CpEntityManagerFactory.java | 43 +++--- .../corepersistence/CpManagerCache.java | 10 +- .../corepersistence/CpRelationManager.java | 68 +++------ .../usergrid/corepersistence/ManagerCache.java | 4 +- .../events/EntityDeletedHandler.java | 29 ++-- .../events/EntityVersionCreatedHandler.java | 12 +- .../events/EntityVersionDeletedHandler.java | 44 +++--- .../results/FilteringLoader.java | 11 +- .../rx/impl/AllApplicationsObservableImpl.java | 7 +- .../rx/impl/AllEntitiesInSystemImpl.java | 13 +- .../corepersistence/util/CpNamingUtils.java | 12 -- .../usergrid/persistence/EntityManager.java | 6 +- .../persistence/EntityManagerFactory.java | 4 +- .../cassandra/EntityManagerFactoryImpl.java | 7 + .../cassandra/EntityManagerImpl.java | 6 +- .../corepersistence/StaleIndexCleanupTest.java | 11 +- .../migration/EntityTypeMappingMigrationIT.java | 10 +- .../org/apache/usergrid/rest/RootResource.java | 6 +- .../applications/ApplicationResource.java | 2 +- 20 files changed, 174 insertions(+), 281 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/abbdd6b4/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java ---------------------------------------------------------------------- diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java index f420d7b..c45c390 100644 --- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java +++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java @@ -33,11 +33,6 @@ import java.util.TreeMap; import java.util.TreeSet; import java.util.UUID; -import com.codahale.metrics.Meter; -import org.apache.usergrid.persistence.collection.FieldSet; -import org.apache.usergrid.persistence.core.future.BetterFuture; -import org.apache.usergrid.persistence.index.ApplicationEntityIndex; -import org.elasticsearch.action.ListenableActionFuture; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.util.Assert; @@ -59,10 +54,6 @@ import org.apache.usergrid.persistence.RelationManager; import org.apache.usergrid.persistence.Results; import org.apache.usergrid.persistence.Schema; import org.apache.usergrid.persistence.SimpleEntityRef; - -import static org.apache.usergrid.corepersistence.util.CpEntityMapUtils.entityToCpEntity; -import static org.apache.usergrid.persistence.SimpleEntityRef.getUuid; - import org.apache.usergrid.persistence.SimpleRoleRef; import org.apache.usergrid.persistence.TypedEntity; import org.apache.usergrid.persistence.cassandra.ApplicationCF; @@ -72,8 +63,8 @@ import org.apache.usergrid.persistence.cassandra.ConnectionRefImpl; import org.apache.usergrid.persistence.cassandra.CounterUtils; import org.apache.usergrid.persistence.cassandra.GeoIndexManager; import org.apache.usergrid.persistence.cassandra.util.TraceParticipant; -import org.apache.usergrid.persistence.collection.CollectionScope; import org.apache.usergrid.persistence.collection.EntityCollectionManager; +import org.apache.usergrid.persistence.collection.FieldSet; import org.apache.usergrid.persistence.collection.exception.WriteOptimisticVerifyException; import org.apache.usergrid.persistence.collection.exception.WriteUniqueVerifyException; import org.apache.usergrid.persistence.core.metrics.MetricsFactory; @@ -88,6 +79,7 @@ import org.apache.usergrid.persistence.exceptions.DuplicateUniquePropertyExistsE import org.apache.usergrid.persistence.exceptions.EntityNotFoundException; import org.apache.usergrid.persistence.exceptions.RequiredPropertyNotFoundException; import org.apache.usergrid.persistence.exceptions.UnexpectedEntityTypeException; +import org.apache.usergrid.persistence.index.ApplicationEntityIndex; import org.apache.usergrid.persistence.index.EntityIndex; import org.apache.usergrid.persistence.index.EntityIndexBatch; import org.apache.usergrid.persistence.index.IndexScope; @@ -105,9 +97,11 @@ import org.apache.usergrid.persistence.model.field.StringField; import org.apache.usergrid.persistence.model.util.UUIDGenerator; import org.apache.usergrid.utils.ClassUtils; import org.apache.usergrid.utils.CompositeUtils; +import org.apache.usergrid.utils.Inflector; import org.apache.usergrid.utils.StringUtils; import org.apache.usergrid.utils.UUIDUtils; +import com.codahale.metrics.Meter; import com.codahale.metrics.Timer; import com.google.common.base.Preconditions; import com.netflix.hystrix.exception.HystrixRuntimeException; @@ -134,6 +128,7 @@ import static me.prettyprint.hector.api.factory.HFactory.createCounterSliceQuery import static me.prettyprint.hector.api.factory.HFactory.createMutator; import static org.apache.commons.lang.StringUtils.capitalize; import static org.apache.commons.lang.StringUtils.isBlank; +import static org.apache.usergrid.corepersistence.util.CpEntityMapUtils.entityToCpEntity; import static org.apache.usergrid.corepersistence.util.CpNamingUtils.getCollectionScopeNameFromEntityType; import static org.apache.usergrid.persistence.Schema.COLLECTION_ROLES; import static org.apache.usergrid.persistence.Schema.COLLECTION_USERS; @@ -150,6 +145,7 @@ import static org.apache.usergrid.persistence.Schema.PROPERTY_TYPE; import static org.apache.usergrid.persistence.Schema.PROPERTY_UUID; import static org.apache.usergrid.persistence.Schema.TYPE_APPLICATION; import static org.apache.usergrid.persistence.Schema.TYPE_ENTITY; +import static org.apache.usergrid.persistence.SimpleEntityRef.getUuid; import static org.apache.usergrid.persistence.cassandra.ApplicationCF.APPLICATION_AGGREGATE_COUNTERS; import static org.apache.usergrid.persistence.cassandra.ApplicationCF.ENTITY_COMPOSITE_DICTIONARIES; import static org.apache.usergrid.persistence.cassandra.ApplicationCF.ENTITY_COUNTERS; @@ -212,7 +208,9 @@ public class CpEntityManager implements EntityManager { private Timer entGetRepairedEntityTimer; private Timer updateEntityTimer; private Meter updateEntityMeter; - private EntityIndex ei; +// private EntityIndex ei; + + private EntityCollectionManager ecm; // /** Short-term cache to keep us from reloading same Entity during single request. */ // private LoadingCache<EntityScope, org.apache.usergrid.persistence.model.entity.Entity> entityCache; @@ -223,8 +221,7 @@ public class CpEntityManager implements EntityManager { } @Override - public void init( EntityManagerFactory emf, EntityIndex ei, UUID applicationId ) { - this.ei = ei; + public void init( EntityManagerFactory emf, UUID applicationId ) { Preconditions.checkNotNull( emf, "emf must not be null" ); Preconditions.checkNotNull( applicationId, "applicationId must not be null" ); @@ -235,6 +232,8 @@ public class CpEntityManager implements EntityManager { applicationScope = CpNamingUtils.getApplicationScope( applicationId ); + ecm = managerCache.getEntityCollectionManager( applicationScope ); + this.cass = this.emf.getCassandraService(); this.counterUtils = this.emf.getCounterUtils(); @@ -287,36 +286,18 @@ public class CpEntityManager implements EntityManager { } - @Override - public Health getIndexHealth() { - return ei.getIndexHealth(); - } - - - /** Needed to support short-term Entity cache. */ - public static class EntityScope { - CollectionScope scope; - Id entityId; - - - public EntityScope( CollectionScope scope, Id entityId ) { - this.scope = scope; - this.entityId = entityId; - } - } /** * Load entity from short-term cache. Package scope so that CpRelationManager can use it too. * - * @param es Carries Entity Id and CollectionScope from which to load Entity. + * @param entityId Load the entity by entityId * * @return Entity or null if not found */ - org.apache.usergrid.persistence.model.entity.Entity load( EntityScope es ) { + org.apache.usergrid.persistence.model.entity.Entity load( Id entityId ) { - return managerCache.getEntityCollectionManager(es.scope) - .load(es.entityId).toBlocking() + return ecm .load( entityId ).toBlocking() .lastOrDefault(null); } @@ -435,23 +416,19 @@ public class CpEntityManager implements EntityManager { Id id = new SimpleId( entityRef.getUuid(), entityRef.getType() ); - CollectionScope collectionScope = getCollectionScopeNameFromEntityType( - getApplicationScope().getApplication(), entityRef.getType()); - // if ( !UUIDUtils.isTimeBased( id.getUuid() ) ) { // throw new IllegalArgumentException( // "Entity Id " + id.getType() + ":"+ id.getUuid() +" uuid not time based"); // } - org.apache.usergrid.persistence.model.entity.Entity cpEntity = load( new EntityScope( collectionScope, id ) ); + org.apache.usergrid.persistence.model.entity.Entity cpEntity = load( id ); if ( cpEntity == null ) { if ( logger.isDebugEnabled() ) { - logger.debug( "FAILED to load entity {}:{} from scope\n app {}\n owner {}\n name {}", + logger.debug( "FAILED to load entity {}:{} from app {}", new Object[] { - id.getType(), id.getUuid(), collectionScope.getApplication(), - collectionScope.getOwner(), collectionScope.getName() + id.getType(), id.getUuid(), applicationId } ); } return null; @@ -518,23 +495,19 @@ public class CpEntityManager implements EntityManager { Id id = new SimpleId( entityId, type ); - CollectionScope collectionScope = getCollectionScopeNameFromEntityType( - getApplicationScope().getApplication(), type); - // if ( !UUIDUtils.isTimeBased( id.getUuid() ) ) { // throw new IllegalArgumentException( // "Entity Id " + id.getType() + ":"+ id.getUuid() +" uuid not time based"); // } - org.apache.usergrid.persistence.model.entity.Entity cpEntity = load( new EntityScope( collectionScope, id ) ); + org.apache.usergrid.persistence.model.entity.Entity cpEntity = load( id ); if ( cpEntity == null ) { if ( logger.isDebugEnabled() ) { - logger.debug( "FAILED to load entity {}:{} from scope\n app {}\n owner {}\n name {}", + logger.debug( "FAILED to load entity {}:{} from app {}\n", new Object[] { - id.getType(), id.getUuid(), collectionScope.getApplication(), - collectionScope.getOwner(), collectionScope.getName() + id.getType(), id.getUuid(), applicationId } ); } return null; @@ -587,18 +560,16 @@ public class CpEntityManager implements EntityManager { updateEntityMeter.mark(); Timer.Context timer = updateEntityTimer.time(); - CollectionScope collectionScope = getCollectionScopeNameFromEntityType(appId, type ); - EntityCollectionManager ecm = managerCache.getEntityCollectionManager( collectionScope ); + + Id entityId = new SimpleId( entity.getUuid(), entity.getType() ); if ( logger.isDebugEnabled() ) { - logger.debug( "Updating entity {}:{} from scope\n app {}\n owner {}\n name {}", + logger.debug( "Updating entity {}:{} app {}\n", new Object[] { entityId.getType(), entityId.getUuid(), - collectionScope.getApplication(), - collectionScope.getOwner(), - collectionScope.getName() + appId } ); } @@ -662,10 +633,8 @@ public class CpEntityManager implements EntityManager { if(applicationScope == null || entityRef == null){ return Observable.empty(); } - CollectionScope collectionScope = getCollectionScopeNameFromEntityType( - getApplicationScope().getApplication(), entityRef.getType() ); - EntityCollectionManager ecm = managerCache.getEntityCollectionManager( collectionScope ); + EntityCollectionManager ecm = managerCache.getEntityCollectionManager( applicationScope ); Id entityId = new SimpleId( entityRef.getUuid(), entityRef.getType() ); @@ -675,7 +644,7 @@ public class CpEntityManager implements EntityManager { // } org.apache.usergrid.persistence.model.entity.Entity entity = - load( new EntityScope( collectionScope, entityId ) ); + load( entityId ); if ( entity != null ) { @@ -825,19 +794,16 @@ public class CpEntityManager implements EntityManager { public Entity getUniqueEntityFromAlias( String collectionType, String aliasType ){ String collName = Schema.defaultCollectionName( collectionType ); - CollectionScope collectionScope = getCollectionScopeNameFromEntityType( - getApplicationScope().getApplication(), collName); String propertyName = Schema.getDefaultSchema().aliasProperty( collName ); Timer.Context repairedEntityGet = entGetRepairedEntityTimer.time(); - final EntityCollectionManager ecm = managerCache.getEntityCollectionManager( collectionScope ); //TODO: can't we just sub in the getEntityRepair method here so for every read of a uniqueEntityField we can verify it is correct? StringField uniqueLookupRepairField = new StringField( propertyName, aliasType.toString()); - Observable<FieldSet> fieldSetObservable = ecm.getEntitiesFromFields( + Observable<FieldSet> fieldSetObservable = ecm.getEntitiesFromFields( Inflector.getInstance().singularize( collectionType ), Arrays.<Field>asList( uniqueLookupRepairField ) ); if(fieldSetObservable == null){ @@ -1074,14 +1040,9 @@ public class CpEntityManager implements EntityManager { @Override public void deleteProperty( EntityRef entityRef, String propertyName ) throws Exception { - CollectionScope collectionScope = getCollectionScopeNameFromEntityType( - getApplicationScope().getApplication(), entityRef.getType()); - IndexScope defaultIndexScope = new IndexScopeImpl( getApplicationScope().getApplication(), getCollectionScopeNameFromEntityType( entityRef.getType() ) ); - EntityCollectionManager ecm = managerCache.getEntityCollectionManager( collectionScope ); - ApplicationEntityIndex aei = managerCache.getEntityIndex( getApplicationScope() ); Id entityId = new SimpleId( entityRef.getUuid(), entityRef.getType() ); @@ -1091,7 +1052,7 @@ public class CpEntityManager implements EntityManager { // } org.apache.usergrid.persistence.model.entity.Entity cpEntity = - load( new EntityScope( collectionScope, entityId ) ); + load( entityId ); cpEntity.removeField( propertyName ); @@ -1103,13 +1064,11 @@ public class CpEntityManager implements EntityManager { cpEntity = ecm.write( cpEntity ).toBlocking().last(); logger.debug( "Wrote {}:{} version {}", new Object[] { - cpEntity.getId().getType(), cpEntity.getId().getUuid(), cpEntity.getVersion() + cpEntity.getId().getType(), cpEntity.getId().getUuid(), cpEntity.getVersion() } ); //Adding graphite metrics - Timer.Context timeESBatch = esDeletePropertyTimer.time(); - BetterFuture future = aei.createBatch().index( defaultIndexScope, cpEntity ).execute(); - timeESBatch.stop(); + // update in all containing collections and connection indexes CpRelationManager rm = ( CpRelationManager ) getRelationManager( entityRef ); rm.updateContainingCollectionAndCollectionIndexes( cpEntity ); @@ -1226,7 +1185,7 @@ public class CpEntityManager implements EntityManager { List<HColumn<ByteBuffer, ByteBuffer>> results = cass.getAllColumns( cass.getApplicationKeyspace( applicationId ), dictionaryCf, - CassandraPersistenceUtils.key( entity.getUuid(), dictionaryName ), be, be ); + CassandraPersistenceUtils.key( entity.getUuid(), dictionaryName ), be, be ); for ( HColumn<ByteBuffer, ByteBuffer> result : results ) { Object name = null; if ( entityHasDictionary ) { @@ -1290,9 +1249,9 @@ public class CpEntityManager implements EntityManager { HColumn<ByteBuffer, ByteBuffer> result = cass.getColumn( cass.getApplicationKeyspace( applicationId ), dictionaryCf, - CassandraPersistenceUtils.key( entity.getUuid(), dictionaryName ), - entityHasDictionary ? bytebuffer( elementName ) : DynamicComposite.toByteBuffer( elementName ), - be, be ); + CassandraPersistenceUtils.key( entity.getUuid(), dictionaryName ), + entityHasDictionary ? bytebuffer( elementName ) : DynamicComposite.toByteBuffer( elementName ), be, + be ); if ( result != null ) { if ( entityHasDictionary && coTypeIsBasic ) { @@ -1338,7 +1297,7 @@ public class CpEntityManager implements EntityManager { ColumnSlice<ByteBuffer, ByteBuffer> results = cass.getColumns( cass.getApplicationKeyspace( applicationId ), dictionaryCf, - CassandraPersistenceUtils.key( entity.getUuid(), dictionaryName ), columnNames, be, be ); + CassandraPersistenceUtils.key( entity.getUuid(), dictionaryName ), columnNames, be, be ); if ( results != null ) { values = new HashMap<String, Object>(); for ( HColumn<ByteBuffer, ByteBuffer> result : results.getColumns() ) { @@ -1430,7 +1389,7 @@ public class CpEntityManager implements EntityManager { throws Exception { return getRelationManager( get( entityId )) - .getCollection ( collectionName, query, resultsLevel ); + .getCollection( collectionName, query, resultsLevel ); } @@ -1496,7 +1455,7 @@ public class CpEntityManager implements EntityManager { public ConnectionRef createConnection( ConnectionRef connection ) throws Exception { return createConnection( connection.getConnectingEntity(), connection.getConnectionType(), - connection.getConnectedEntity() ); + connection.getConnectedEntity() ); } @@ -1836,7 +1795,7 @@ public class CpEntityManager implements EntityManager { .addToCollection( COLLECTION_ROLES, entity ); logger.info( "Created role {} with id {} in group {}", - new String[] { roleName, entity.getUuid().toString(), groupId.toString() } ); + new String[] { roleName, entity.getUuid().toString(), groupId.toString() } ); return entity; } @@ -1989,7 +1948,7 @@ public class CpEntityManager implements EntityManager { private EntityRef getRoleRef( String roleName ) throws Exception { Results results = this.searchCollection( new SimpleEntityRef( Application.ENTITY_TYPE, applicationId ), - Schema.defaultCollectionName( Role.ENTITY_TYPE ), Query.findForProperty( "roleName", roleName ) ); + Schema.defaultCollectionName( Role.ENTITY_TYPE ), Query.findForProperty( "roleName", roleName ) ); Iterator<Entity> iterator = results.iterator(); EntityRef roleRef = null; while ( iterator.hasNext() ) { @@ -2297,13 +2256,8 @@ public class CpEntityManager implements EntityManager { private Id getIdForUniqueEntityField( final String collectionName, final String propertyName, final Object propertyValue ) { - CollectionScope collectionScope = getCollectionScopeNameFromEntityType( - getApplicationScope().getApplication(), collectionName); - - final EntityCollectionManager ecm = managerCache.getEntityCollectionManager( collectionScope ); - //convert to a string, that's what we store - final Id results = ecm.getIdField( new StringField( + final Id results = ecm.getIdField( Inflector.getInstance().singularize( collectionName ), new StringField( propertyName, propertyValue.toString() ) ).toBlocking() .lastOrDefault( null ); return results; } @@ -2621,18 +2575,13 @@ public class CpEntityManager implements EntityManager { org.apache.usergrid.persistence.model.entity.Entity cpEntity = entityToCpEntity( entity, importId ); // prepare to write and index Core Persistence Entity into default scope - CollectionScope collectionScope = getCollectionScopeNameFromEntityType(getApplicationScope().getApplication(), eType); - - EntityCollectionManager ecm = managerCache.getEntityCollectionManager( collectionScope ); if ( logger.isDebugEnabled() ) { - logger.debug( "Writing entity {}:{} into scope\n app {}\n owner {}\n name {} data {}", + logger.debug( "Writing entity {}:{} into app {}\n", new Object[] { entity.getType(), entity.getUuid(), - collectionScope.getApplication(), - collectionScope.getOwner(), - collectionScope.getName(), + applicationId, CpEntityMapUtils.toMap( cpEntity ) } ); // @@ -2646,15 +2595,20 @@ public class CpEntityManager implements EntityManager { } try { - logger.debug( "About to Write {}:{} version {}", new Object[] { + + if(logger.isDebugEnabled()) { + logger.debug( "About to Write {}:{} version {}", new Object[] { cpEntity.getId().getType(), cpEntity.getId().getUuid(), cpEntity.getVersion() - } ); + } ); + } cpEntity = ecm .write( cpEntity ).toBlocking().last(); - logger.debug( "Wrote {}:{} version {}", new Object[] { + if(logger.isDebugEnabled()) { + logger.debug( "Wrote {}:{} version {}", new Object[] { cpEntity.getId().getType(), cpEntity.getId().getUuid(), cpEntity.getVersion() - } ); + } ); + } } catch ( WriteUniqueVerifyException wuve ) { http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/abbdd6b4/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java ---------------------------------------------------------------------- diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java index 3cfc1a4..9cd1714 100644 --- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java +++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java @@ -25,9 +25,6 @@ import java.util.TreeMap; import java.util.UUID; import java.util.concurrent.atomic.AtomicBoolean; -import com.hazelcast.core.IdGenerator; -import org.apache.usergrid.persistence.index.ApplicationEntityIndex; -import org.apache.usergrid.persistence.index.EntityIndexFactory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.BeansException; @@ -47,9 +44,7 @@ import org.apache.usergrid.persistence.Results; import org.apache.usergrid.persistence.cassandra.CassandraService; import org.apache.usergrid.persistence.cassandra.CounterUtils; import org.apache.usergrid.persistence.cassandra.Setup; -import org.apache.usergrid.persistence.collection.CollectionScope; import org.apache.usergrid.persistence.collection.EntityCollectionManager; -import org.apache.usergrid.persistence.collection.impl.CollectionScopeImpl; import org.apache.usergrid.persistence.collection.serialization.impl.migration.EntityIdScope; import org.apache.usergrid.persistence.core.metrics.MetricsFactory; import org.apache.usergrid.persistence.core.migration.data.MigrationDataProvider; @@ -64,7 +59,9 @@ import org.apache.usergrid.persistence.graph.Edge; import org.apache.usergrid.persistence.graph.GraphManager; import org.apache.usergrid.persistence.graph.SearchByEdgeType; import org.apache.usergrid.persistence.graph.impl.SimpleSearchByEdgeType; +import org.apache.usergrid.persistence.index.ApplicationEntityIndex; import org.apache.usergrid.persistence.index.EntityIndex; +import org.apache.usergrid.persistence.index.EntityIndexFactory; import org.apache.usergrid.persistence.index.query.Query; import org.apache.usergrid.persistence.model.entity.Id; import org.apache.usergrid.persistence.model.entity.SimpleId; @@ -205,7 +202,7 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application private EntityManager _getEntityManager( UUID applicationId ) { EntityManager em = new CpEntityManager(); - em.init( this,entityIndex ,applicationId ); + em.init( this ,applicationId ); return em; } @@ -437,20 +434,20 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application Map<String, UUID> appMap = new HashMap<String, UUID>(); ApplicationScope appScope = CpNamingUtils.getApplicationScope( CpNamingUtils.SYSTEM_APP_ID ); - GraphManager gm = managerCache.getGraphManager(appScope); + GraphManager gm = managerCache.getGraphManager( appScope ); - EntityManager em = getEntityManager( CpNamingUtils.SYSTEM_APP_ID); + EntityManager em = getEntityManager( CpNamingUtils.SYSTEM_APP_ID ); Application app = em.getApplication(); Id fromEntityId = new SimpleId( app.getUuid(), app.getType() ); - final String scopeName; +// final String scopeName; final String edgeType; if ( deleted ) { - edgeType = CpNamingUtils.getEdgeTypeFromCollectionName(CpNamingUtils.DELETED_APPINFOS); - scopeName = CpNamingUtils.getCollectionScopeNameFromCollectionName(CpNamingUtils.DELETED_APPINFOS); + edgeType = CpNamingUtils.getEdgeTypeFromCollectionName( CpNamingUtils.DELETED_APPINFOS ); +// scopeName = CpNamingUtils.getCollectionScopeNameFromCollectionName(CpNamingUtils.DELETED_APPINFOS); } else { - edgeType = CpNamingUtils.getEdgeTypeFromCollectionName(CpNamingUtils.APPINFOS); - scopeName = CpNamingUtils.getCollectionScopeNameFromCollectionName(CpNamingUtils.APPINFOS); + edgeType = CpNamingUtils.getEdgeTypeFromCollectionName( CpNamingUtils.APPINFOS ); +// scopeName = CpNamingUtils.getCollectionScopeNameFromCollectionName(CpNamingUtils.APPINFOS); } logger.debug("getApplications(): Loading edges of edgeType {} from {}:{}", @@ -472,13 +469,10 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application edge.getTargetNode().getType(), edge.getTargetNode().getUuid() }); - CollectionScope collScope = new CollectionScopeImpl( - appScope.getApplication(), - appScope.getApplication(), - scopeName); + org.apache.usergrid.persistence.model.entity.Entity e = - managerCache.getEntityCollectionManager( collScope ).load( targetId ) + managerCache.getEntityCollectionManager( appScope ).load( targetId ) .toBlocking().lastOrDefault(null); if ( e == null ) { @@ -777,13 +771,16 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application // could use any collection scope here, does not matter EntityCollectionManager ecm = getManagerCache().getEntityCollectionManager( - new CollectionScopeImpl( - new SimpleId( CpNamingUtils.SYSTEM_APP_ID, "application"), - new SimpleId( CpNamingUtils.SYSTEM_APP_ID, "application"), - "dummy" - )); + new ApplicationScopeImpl( + new SimpleId( CpNamingUtils.SYSTEM_APP_ID, "application")) + ); return ecm.getHealth(); } + + @Override + public Health getIndexHealth() { + return entityIndex.getIndexHealth(); + } } http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/abbdd6b4/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpManagerCache.java ---------------------------------------------------------------------- diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpManagerCache.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpManagerCache.java index 4e35f8c..d057fd6 100644 --- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpManagerCache.java +++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpManagerCache.java @@ -16,25 +16,17 @@ package org.apache.usergrid.corepersistence; -import java.util.concurrent.ExecutionException; - -import com.amazonaws.services.cloudfront.model.InvalidArgumentException; -import org.apache.usergrid.persistence.collection.CollectionScope; import org.apache.usergrid.persistence.collection.EntityCollectionManager; import org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory; import org.apache.usergrid.persistence.core.scope.ApplicationScope; import org.apache.usergrid.persistence.graph.GraphManager; import org.apache.usergrid.persistence.graph.GraphManagerFactory; import org.apache.usergrid.persistence.index.ApplicationEntityIndex; -import org.apache.usergrid.persistence.index.EntityIndex; import org.apache.usergrid.persistence.index.EntityIndexFactory; import org.apache.usergrid.persistence.map.MapManager; import org.apache.usergrid.persistence.map.MapManagerFactory; import org.apache.usergrid.persistence.map.MapScope; -import com.google.common.cache.CacheBuilder; -import com.google.common.cache.CacheLoader; -import com.google.common.cache.LoadingCache; import com.google.inject.Inject; @@ -62,7 +54,7 @@ public class CpManagerCache implements ManagerCache { @Override - public EntityCollectionManager getEntityCollectionManager( CollectionScope scope ) { + public EntityCollectionManager getEntityCollectionManager( ApplicationScope scope ) { //cache is now in the colletion manager level return ecmf.createCollectionManager( scope ); } http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/abbdd6b4/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java ---------------------------------------------------------------------- diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java index 5416f19..3427684 100644 --- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java +++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java @@ -29,16 +29,14 @@ import java.util.Map; import java.util.Set; import java.util.UUID; -import org.apache.usergrid.persistence.core.future.BetterFuture; -import org.apache.usergrid.persistence.index.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.util.Assert; +import org.apache.usergrid.corepersistence.results.CollectionResultsLoaderFactoryImpl; import org.apache.usergrid.corepersistence.results.ConnectionResultsLoaderFactoryImpl; import org.apache.usergrid.corepersistence.results.ElasticSearchQueryExecutor; import org.apache.usergrid.corepersistence.results.QueryExecutor; -import org.apache.usergrid.corepersistence.results.CollectionResultsLoaderFactoryImpl; import org.apache.usergrid.corepersistence.util.CpEntityMapUtils; import org.apache.usergrid.corepersistence.util.CpNamingUtils; import org.apache.usergrid.persistence.ConnectedEntityRef; @@ -62,7 +60,6 @@ import org.apache.usergrid.persistence.cassandra.index.ConnectedIndexScanner; import org.apache.usergrid.persistence.cassandra.index.IndexBucketScanner; import org.apache.usergrid.persistence.cassandra.index.IndexScanner; import org.apache.usergrid.persistence.cassandra.index.NoOpIndexScanner; -import org.apache.usergrid.persistence.collection.CollectionScope; import org.apache.usergrid.persistence.core.future.BetterFuture; import org.apache.usergrid.persistence.core.metrics.MetricsFactory; import org.apache.usergrid.persistence.core.scope.ApplicationScope; @@ -80,6 +77,8 @@ import org.apache.usergrid.persistence.graph.impl.SimpleSearchByEdgeType; import org.apache.usergrid.persistence.graph.impl.SimpleSearchEdgeType; import org.apache.usergrid.persistence.index.ApplicationEntityIndex; import org.apache.usergrid.persistence.index.EntityIndexBatch; +import org.apache.usergrid.persistence.index.IndexScope; +import org.apache.usergrid.persistence.index.SearchTypes; import org.apache.usergrid.persistence.index.impl.IndexScopeImpl; import org.apache.usergrid.persistence.index.query.Identifier; import org.apache.usergrid.persistence.index.query.Query; @@ -114,13 +113,11 @@ import me.prettyprint.hector.api.beans.HColumn; import me.prettyprint.hector.api.mutation.Mutator; import rx.Observable; import rx.functions.Action1; -import rx.functions.Action2; import rx.functions.Func1; import static java.util.Arrays.asList; import static me.prettyprint.hector.api.factory.HFactory.createMutator; -import static org.apache.usergrid.corepersistence.util.CpNamingUtils.getCollectionScopeNameFromEntityType; import static org.apache.usergrid.persistence.Schema.COLLECTION_ROLES; import static org.apache.usergrid.persistence.Schema.DICTIONARY_CONNECTED_ENTITIES; import static org.apache.usergrid.persistence.Schema.DICTIONARY_CONNECTED_TYPES; @@ -180,7 +177,6 @@ public class CpRelationManager implements RelationManager { private org.apache.usergrid.persistence.model.entity.Entity cpHeadEntity; private ApplicationScope applicationScope; - private CollectionScope headEntityScope; private CassandraService cass; @@ -228,19 +224,16 @@ public class CpRelationManager implements RelationManager { .getTimer( CpRelationManager.class, "relation.manager.es.create.connection.timer" ); this.cassConnectionDelete = metricsFactory .getTimer( CpRelationManager.class, "relation.manager.cassandra.delete.connection.batch.timer" ); - this.esDeleteConnectionTimer = metricsFactory.getTimer(CpRelationManager.class, "relation.manager.es.delete.connection.batch.timer" ); - // load the Core Persistence version of the head entity as well - this.headEntityScope = getCollectionScopeNameFromEntityType( - applicationScope.getApplication(), headEntity.getType()); + this.esDeleteConnectionTimer = metricsFactory.getTimer( CpRelationManager.class, + "relation.manager.es.delete.connection.batch.timer" ); + if ( logger.isDebugEnabled() ) { - logger.debug( "Loading head entity {}:{} from scope\n app {}\n owner {}\n name {}", + logger.debug( "Loading head entity {}:{} from app {}", new Object[] { headEntity.getType(), headEntity.getUuid(), - headEntityScope.getApplication(), - headEntityScope.getOwner(), - headEntityScope.getName() + applicationScope } ); } @@ -250,7 +243,7 @@ public class CpRelationManager implements RelationManager { // cpHeadEntity = entityToCpEntity( (Entity)headEntity, headEntity.getUuid() ); // }else { this.cpHeadEntity = - ( ( CpEntityManager ) em ).load( new CpEntityManager.EntityScope( headEntityScope, entityId ) ); + ( ( CpEntityManager ) em ).load( entityId ); // } // commented out because it is possible that CP entity has not been created yet @@ -609,12 +602,9 @@ public class CpRelationManager implements RelationManager { public Entity addToCollection( String collName, EntityRef itemRef, boolean connectBack ) throws Exception { - CollectionScope memberScope = getCollectionScopeNameFromEntityType( - applicationScope.getApplication(), itemRef.getType()); - Id entityId = new SimpleId( itemRef.getUuid(), itemRef.getType() ); org.apache.usergrid.persistence.model.entity.Entity memberEntity = - ((CpEntityManager)em).load( new CpEntityManager.EntityScope( memberScope, entityId)); + ((CpEntityManager)em).load( entityId ); return addToCollection(collName, itemRef, memberEntity, connectBack); } @@ -642,9 +632,7 @@ public class CpRelationManager implements RelationManager { return null; } - // load the new member entity to be added to the collection from its default scope - CollectionScope memberScope = getCollectionScopeNameFromEntityType( - applicationScope.getApplication(), itemRef.getType()); + if ( memberEntity == null ) { throw new RuntimeException( @@ -652,14 +640,11 @@ public class CpRelationManager implements RelationManager { } if ( logger.isDebugEnabled() ) { - logger.debug( "Loaded member entity {}:{} from scope\n app {}\n " - + "owner {}\n name {} data {}", + logger.debug( "Loaded member entity {}:{} from app {}\n " + + " data {}", new Object[] { itemRef.getType(), - itemRef.getUuid(), - memberScope.getApplication(), - memberScope.getOwner(), - memberScope.getName(), + itemRef.getUuid(),applicationScope, CpEntityMapUtils.toMap( memberEntity ) } ); } @@ -791,24 +776,21 @@ public class CpRelationManager implements RelationManager { } // load the entity to be removed to the collection - CollectionScope memberScope = getCollectionScopeNameFromEntityType( - applicationScope.getApplication(), itemRef.getType()); + if ( logger.isDebugEnabled() ) { logger.debug( "Loading entity to remove from collection " - + "{}:{} from scope\n app {}\n owner {}\n name {}", + + "{}:{} from app {}\n", new Object[] { itemRef.getType(), itemRef.getUuid(), - memberScope.getApplication(), - memberScope.getOwner(), - memberScope.getName() + applicationScope }); } Id entityId = new SimpleId( itemRef.getUuid(), itemRef.getType() ); org.apache.usergrid.persistence.model.entity.Entity memberEntity = - ((CpEntityManager)em).load( new CpEntityManager.EntityScope( memberScope, entityId)); + ((CpEntityManager)em).load( entityId ); final ApplicationEntityIndex ei = managerCache.getEntityIndex( applicationScope ); final EntityIndexBatch batch = ei.createBatch(); @@ -964,28 +946,24 @@ public class CpRelationManager implements RelationManager { ConnectionRefImpl connection = new ConnectionRefImpl( headEntity, connectionType, connectedEntityRef ); - CollectionScope targetScope = getCollectionScopeNameFromEntityType( - applicationScope.getApplication(), connectedEntityRef.getType()); if ( logger.isDebugEnabled() ) { logger.debug("createConnection(): " + "Indexing connection type '{}'\n from source {}:{}]\n" - + " to target {}:{}\n from scope\n app {}\n owner {}\n name {}", + + " to target {}:{}\n app {}", new Object[] { connectionType, headEntity.getType(), headEntity.getUuid(), connectedEntityRef.getType(), connectedEntityRef.getUuid(), - targetScope.getApplication(), - targetScope.getOwner(), - targetScope.getName() + applicationScope }); } Id entityId = new SimpleId( connectedEntityRef.getUuid(), connectedEntityRef.getType()); org.apache.usergrid.persistence.model.entity.Entity targetEntity = - ((CpEntityManager)em).load( new CpEntityManager.EntityScope( targetScope, entityId)); + ((CpEntityManager)em).load( entityId ); String edgeType = CpNamingUtils.getEdgeTypeFromConnectionType( connectionType ); @@ -1199,8 +1177,6 @@ public class CpRelationManager implements RelationManager { String connectionType = connectionRef.getConnectedEntity().getConnectionType(); - CollectionScope targetScope = getCollectionScopeNameFromEntityType( applicationScope.getApplication(), - connectedEntityRef.getType() ); if ( logger.isDebugEnabled() ) { logger.debug( "Deleting connection '{}' from source {}:{} \n to target {}:{}", @@ -1215,7 +1191,7 @@ public class CpRelationManager implements RelationManager { Id entityId = new SimpleId( connectedEntityRef.getUuid(), connectedEntityRef.getType() ); org.apache.usergrid.persistence.model.entity.Entity targetEntity = - ((CpEntityManager)em).load( new CpEntityManager.EntityScope( targetScope, entityId)); + ((CpEntityManager)em).load( entityId ); // Delete graph edge connection from head entity to member entity Edge edge = new SimpleEdge( http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/abbdd6b4/stack/core/src/main/java/org/apache/usergrid/corepersistence/ManagerCache.java ---------------------------------------------------------------------- diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/ManagerCache.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/ManagerCache.java index 3b94927..dfa62fb 100644 --- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/ManagerCache.java +++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/ManagerCache.java @@ -20,12 +20,10 @@ package org.apache.usergrid.corepersistence; -import org.apache.usergrid.persistence.collection.CollectionScope; import org.apache.usergrid.persistence.collection.EntityCollectionManager; import org.apache.usergrid.persistence.core.scope.ApplicationScope; import org.apache.usergrid.persistence.graph.GraphManager; import org.apache.usergrid.persistence.index.ApplicationEntityIndex; -import org.apache.usergrid.persistence.index.EntityIndex; import org.apache.usergrid.persistence.map.MapManager; import org.apache.usergrid.persistence.map.MapScope; @@ -40,7 +38,7 @@ public interface ManagerCache { * @param scope * @return */ - EntityCollectionManager getEntityCollectionManager( CollectionScope scope ); + EntityCollectionManager getEntityCollectionManager( ApplicationScope scope ); /** * Get the entity index for the specified app scope http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/abbdd6b4/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityDeletedHandler.java ---------------------------------------------------------------------- diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityDeletedHandler.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityDeletedHandler.java index 85c59da..78c1ca7 100644 --- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityDeletedHandler.java +++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityDeletedHandler.java @@ -20,13 +20,14 @@ package org.apache.usergrid.corepersistence.events; import java.util.UUID; +import org.apache.usergrid.exception.NotImplementedException; +import org.apache.usergrid.persistence.core.scope.ApplicationScope; import org.apache.usergrid.persistence.index.ApplicationEntityIndex; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.usergrid.corepersistence.CpEntityManagerFactory; import org.apache.usergrid.persistence.EntityManagerFactory; -import org.apache.usergrid.persistence.collection.CollectionScope; import org.apache.usergrid.persistence.collection.event.EntityDeleted; import org.apache.usergrid.persistence.index.EntityIndex; import org.apache.usergrid.persistence.index.IndexScope; @@ -56,7 +57,7 @@ public class EntityDeletedHandler implements EntityDeleted { @Override - public void deleted( CollectionScope scope, Id entityId, UUID version ) { + public void deleted( ApplicationScope scope, Id entityId, UUID version ) { // This check is for testing purposes and for a test that to be able to dynamically turn // off and on delete previous versions so that it can test clean-up on read. @@ -64,20 +65,26 @@ public class EntityDeletedHandler implements EntityDeleted { return; } - logger.debug( "Handling deleted event for entity {}:{} v {} " + "scope\n name: {}\n owner: {}\n app: {}", - new Object[] { - entityId.getType(), entityId.getUuid(), version, scope.getName(), scope.getOwner(), - scope.getApplication() - } ); + + + if(logger.isDebugEnabled()) { + logger.debug( + "Handling deleted event for entity {}:{} v {} " + " app: {}", + new Object[] { + entityId.getType(), entityId.getUuid(), version, + scope.getApplication() + } ); + } CpEntityManagerFactory cpemf = ( CpEntityManagerFactory ) emf; final ApplicationEntityIndex ei = cpemf.getManagerCache().getEntityIndex( scope ); - final IndexScope indexScope = - new IndexScopeImpl( new SimpleId( scope.getOwner().getUuid(), scope.getOwner().getType() ), - scope.getName() ); + throw new NotImplementedException( "Fix this" ); + + //read all edges to this node and de-index them + - ei.createBatch().deindex( indexScope, entityId, version ).execute(); +// ei.createBatch().deindex( indexScope, entityId, version ).execute(); } } http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/abbdd6b4/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionCreatedHandler.java ---------------------------------------------------------------------- diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionCreatedHandler.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionCreatedHandler.java index 6270501..c000500 100644 --- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionCreatedHandler.java +++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionCreatedHandler.java @@ -17,21 +17,19 @@ */ package org.apache.usergrid.corepersistence.events; -import com.google.inject.Inject; -import com.google.inject.Singleton; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.apache.usergrid.corepersistence.CpEntityManagerFactory; -import static org.apache.usergrid.corepersistence.CoreModule.EVENTS_DISABLED; import org.apache.usergrid.persistence.EntityManagerFactory; -import org.apache.usergrid.persistence.collection.CollectionScope; import org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory; import org.apache.usergrid.persistence.collection.event.EntityVersionCreated; -import org.apache.usergrid.persistence.index.EntityIndex; +import org.apache.usergrid.persistence.core.scope.ApplicationScope; import org.apache.usergrid.persistence.model.entity.Entity; +import com.google.inject.Inject; +import com.google.inject.Singleton; + /** * Clean up stale entity indexes when new version of Entity created. Called when an Entity is @@ -56,7 +54,7 @@ public class EntityVersionCreatedHandler implements EntityVersionCreated { @Override - public void versionCreated( final CollectionScope scope, final Entity entity ) { + public void versionCreated( final ApplicationScope scope, final Entity entity ) { //not op, we're not migrating properly to this. Make this an event // // This check is for testing purposes and for a test that to be able to dynamically turn http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/abbdd6b4/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionDeletedHandler.java ---------------------------------------------------------------------- diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionDeletedHandler.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionDeletedHandler.java index f0dd136..4fa5ce1 100644 --- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionDeletedHandler.java +++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionDeletedHandler.java @@ -24,23 +24,17 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.usergrid.corepersistence.CpEntityManagerFactory; +import org.apache.usergrid.exception.NotImplementedException; import org.apache.usergrid.persistence.EntityManagerFactory; -import org.apache.usergrid.persistence.collection.CollectionScope; import org.apache.usergrid.persistence.collection.MvccLogEntry; import org.apache.usergrid.persistence.collection.event.EntityVersionDeleted; -import org.apache.usergrid.persistence.collection.serialization.SerializationFig; +import org.apache.usergrid.persistence.core.scope.ApplicationScope; import org.apache.usergrid.persistence.index.ApplicationEntityIndex; -import org.apache.usergrid.persistence.index.EntityIndex; -import org.apache.usergrid.persistence.index.IndexScope; -import org.apache.usergrid.persistence.index.impl.IndexScopeImpl; import org.apache.usergrid.persistence.model.entity.Id; -import org.apache.usergrid.persistence.model.entity.SimpleId; import com.google.inject.Inject; import com.google.inject.Singleton; -import rx.Observable; - import static org.apache.usergrid.corepersistence.CoreModule.EVENTS_DISABLED; @@ -62,7 +56,7 @@ public class EntityVersionDeletedHandler implements EntityVersionDeleted { @Override - public void versionDeleted( final CollectionScope scope, final Id entityId, + public void versionDeleted( final ApplicationScope scope, final Id entityId, final List<MvccLogEntry> entityVersions ) { @@ -74,8 +68,8 @@ public class EntityVersionDeletedHandler implements EntityVersionDeleted { if ( logger.isDebugEnabled() ) { logger.debug( "Handling versionDeleted count={} event for entity {}:{} v {} " - + "scope\n name: {}\n owner: {}\n app: {}", new Object[] { - entityVersions.size(), entityId.getType(), entityId.getUuid(), scope.getName(), scope.getOwner(), + + " app: {}", new Object[] { + entityVersions.size(), entityId.getType(), entityId.getUuid(), scope.getApplication() } ); } @@ -84,17 +78,21 @@ public class EntityVersionDeletedHandler implements EntityVersionDeleted { final ApplicationEntityIndex ei = cpemf.getManagerCache().getEntityIndex( scope ); - final IndexScope indexScope = - new IndexScopeImpl( new SimpleId( scope.getOwner().getUuid(), scope.getOwner().getType() ), - scope.getName() ); - - //create our batch, and then collect all of them into a single batch - Observable.from( entityVersions ).collect( () -> ei.createBatch(), ( entityIndexBatch, mvccLogEntry ) -> { - entityIndexBatch.deindex( indexScope, mvccLogEntry.getEntityId(), mvccLogEntry.getVersion() ); - } ) - //after our batch is collected, execute it - .doOnNext( entityIndexBatch -> { - entityIndexBatch.execute(); - } ).toBlocking().last(); + + throw new NotImplementedException( "Fix this" ); + + +// final IndexScope indexScope = +// new IndexScopeImpl( new SimpleId( scope.getOwner().getUuid(), scope.getOwner().getType() ), +// scope.getName() ); +// +// //create our batch, and then collect all of them into a single batch +// Observable.from( entityVersions ).collect( () -> ei.createBatch(), ( entityIndexBatch, mvccLogEntry ) -> { +// entityIndexBatch.deindex( indexScope, mvccLogEntry.getEntityId(), mvccLogEntry.getVersion() ); +// } ) +// //after our batch is collected, execute it +// .doOnNext( entityIndexBatch -> { +// entityIndexBatch.execute(); +// } ).toBlocking().last(); } } http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/abbdd6b4/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/FilteringLoader.java ---------------------------------------------------------------------- diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/FilteringLoader.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/FilteringLoader.java index 0a0f71a..2eb6675 100644 --- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/FilteringLoader.java +++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/FilteringLoader.java @@ -30,18 +30,15 @@ import java.util.UUID; import javax.annotation.Nullable; -import org.apache.usergrid.persistence.index.ApplicationEntityIndex; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.usergrid.corepersistence.ManagerCache; import org.apache.usergrid.corepersistence.util.CpNamingUtils; import org.apache.usergrid.persistence.Results; -import org.apache.usergrid.persistence.collection.CollectionScope; import org.apache.usergrid.persistence.collection.EntityCollectionManager; -import org.apache.usergrid.persistence.collection.impl.CollectionScopeImpl; import org.apache.usergrid.persistence.core.scope.ApplicationScope; -import org.apache.usergrid.persistence.index.EntityIndex; +import org.apache.usergrid.persistence.index.ApplicationEntityIndex; import org.apache.usergrid.persistence.index.EntityIndexBatch; import org.apache.usergrid.persistence.index.IndexScope; import org.apache.usergrid.persistence.index.query.CandidateResult; @@ -211,10 +208,10 @@ public class FilteringLoader implements ResultsLoader { // Get the collection scope and batch load all the versions. We put all entities in // app/app for easy retrieval/ unless persistence changes, we never want to read from // any scope other than the app, app, scope name scope - final CollectionScope collScope = new CollectionScopeImpl( - applicationScope.getApplication(), applicationScope.getApplication(), scopeName); +// final CollectionScope collScope = new CollectionScopeImpl( +// applicationScope.getApplication(), applicationScope.getApplication(), scopeName); - final EntityCollectionManager ecm = managerCache.getEntityCollectionManager( collScope); + final EntityCollectionManager ecm = managerCache.getEntityCollectionManager( applicationScope); //load the results into the loader for this scope for validation http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/abbdd6b4/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllApplicationsObservableImpl.java ---------------------------------------------------------------------- diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllApplicationsObservableImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllApplicationsObservableImpl.java index 0fc5452..0e5fcc5 100644 --- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllApplicationsObservableImpl.java +++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllApplicationsObservableImpl.java @@ -28,10 +28,8 @@ import org.slf4j.LoggerFactory; import org.apache.usergrid.corepersistence.AllApplicationsObservable; import org.apache.usergrid.corepersistence.util.CpNamingUtils; -import org.apache.usergrid.persistence.collection.CollectionScope; import org.apache.usergrid.persistence.collection.EntityCollectionManager; import org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory; -import org.apache.usergrid.persistence.collection.impl.CollectionScopeImpl; import org.apache.usergrid.persistence.core.scope.ApplicationScope; import org.apache.usergrid.persistence.graph.Edge; import org.apache.usergrid.persistence.graph.GraphManager; @@ -81,12 +79,9 @@ public class AllApplicationsObservableImpl implements AllApplicationsObservable final ApplicationScope appScope = getApplicationScope( CpNamingUtils.SYSTEM_APP_ID ); - final CollectionScope appInfoCollectionScope = - new CollectionScopeImpl( appScope.getApplication(), appScope.getApplication(), - CpNamingUtils.getCollectionScopeNameFromCollectionName( CpNamingUtils.APPINFOS ) ); final EntityCollectionManager collectionManager = - entityCollectionManagerFactory.createCollectionManager( appInfoCollectionScope ); + entityCollectionManagerFactory.createCollectionManager( appScope ); final GraphManager gm = graphManagerFactory.createEdgeManager(appScope); http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/abbdd6b4/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllEntitiesInSystemImpl.java ---------------------------------------------------------------------- diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllEntitiesInSystemImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllEntitiesInSystemImpl.java index 758a8aa..83f7d84 100644 --- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllEntitiesInSystemImpl.java +++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllEntitiesInSystemImpl.java @@ -21,8 +21,6 @@ package org.apache.usergrid.corepersistence.rx.impl; import org.apache.usergrid.corepersistence.AllApplicationsObservable; -import org.apache.usergrid.corepersistence.util.CpNamingUtils; -import org.apache.usergrid.persistence.collection.CollectionScope; import org.apache.usergrid.persistence.collection.serialization.impl.migration.EntityIdScope; import org.apache.usergrid.persistence.core.scope.ApplicationScope; import org.apache.usergrid.persistence.graph.GraphManagerFactory; @@ -34,9 +32,8 @@ import com.google.inject.Singleton; /** - * An observable that will emit every entity Id stored in our entire system across all apps. - * Note that this only walks each application applicationId graph, and emits edges from the applicationId and it's edges as the s - * source node + * An observable that will emit every entity Id stored in our entire system across all apps. Note that this only walks + * each application applicationId graph, and emits edges from the applicationId and it's edges as the s source node */ @Singleton public class AllEntitiesInSystemImpl extends AbstractGraphVisitorImpl<EntityIdScope> { @@ -52,11 +49,9 @@ public class AllEntitiesInSystemImpl extends AbstractGraphVisitorImpl<EntityIdSc @Override protected EntityIdScope generateData( final ApplicationScope applicationScope, final Id nodeId ) { - CollectionScope scope = - CpNamingUtils.getCollectionScopeNameFromEntityType( applicationScope.getApplication(), nodeId.getType() ); - final EntityIdScope idScope = new EntityIdScope( scope, nodeId ); + final EntityIdScope idScope = new EntityIdScope( applicationScope, nodeId ); - return idScope; + return idScope; } } http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/abbdd6b4/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/CpNamingUtils.java ---------------------------------------------------------------------- diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/CpNamingUtils.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/CpNamingUtils.java index 58605e0..2e9fb55 100644 --- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/CpNamingUtils.java +++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/CpNamingUtils.java @@ -22,8 +22,6 @@ package org.apache.usergrid.corepersistence.util; import java.util.UUID; import org.apache.usergrid.persistence.Schema; -import org.apache.usergrid.persistence.collection.CollectionScope; -import org.apache.usergrid.persistence.collection.impl.CollectionScopeImpl; import org.apache.usergrid.persistence.core.scope.ApplicationScope; import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl; import org.apache.usergrid.persistence.entities.Application; @@ -31,7 +29,6 @@ import org.apache.usergrid.persistence.map.MapScope; import org.apache.usergrid.persistence.map.impl.MapScopeImpl; import org.apache.usergrid.persistence.model.entity.Id; import org.apache.usergrid.persistence.model.entity.SimpleId; -import rx.functions.Func1; /** @@ -77,15 +74,6 @@ public class CpNamingUtils { - /** - * Generate a collection scope for a collection within the application's Id for the given type - * @param applicationId The applicationId that owns this entity - * @param type The type in the collection - * @return The collectionScope - */ - public static CollectionScope getCollectionScopeNameFromEntityType(final Id applicationId, final String type){ - return new CollectionScopeImpl( applicationId, applicationId, getCollectionScopeNameFromEntityType( type ) ); - } /** * Get the collection name from the entity/id type http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/abbdd6b4/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java ---------------------------------------------------------------------- diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java b/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java index 7108472..d636cfc 100644 --- a/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java +++ b/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java @@ -685,7 +685,7 @@ public interface EntityManager { /** @return the cass */ CassandraService getCass(); - public void init( EntityManagerFactory emf, EntityIndex entityIndex, UUID applicationId); + public void init( EntityManagerFactory emf, UUID applicationId); /** For testing purposes */ public void flushManagerCaches(); @@ -695,10 +695,6 @@ public interface EntityManager { public void reindex( final EntityManagerFactory.ProgressObserver po ) throws Exception; - /** - * Get health status of application's index. - */ - public Health getIndexHealth(); public Entity getUniqueEntityFromAlias( String aliasType, String aliasValue ); } http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/abbdd6b4/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java ---------------------------------------------------------------------- diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java b/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java index 2a74c74..95e7e34 100644 --- a/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java +++ b/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java @@ -147,11 +147,13 @@ public interface EntityManagerFactory { public Health getEntityStoreHealth(); + public Health getIndexHealth(); + void restoreApplication(UUID applicationId) throws Exception; public interface ProgressObserver { - public void onProgress( EntityRef entity); + public void onProgress( EntityRef entity); } } http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/abbdd6b4/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImpl.java ---------------------------------------------------------------------- diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImpl.java b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImpl.java index 0e1e62a..6627fa4 100644 --- a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImpl.java +++ b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImpl.java @@ -444,6 +444,13 @@ public class EntityManagerFactoryImpl implements EntityManagerFactory, Applicati throw new UnsupportedOperationException("Not supported in v1."); } + + @Override + public Health getIndexHealth() { + return null; + } + + @Override public void restoreApplication(UUID applicationId) throws Exception { throw new UnsupportedOperationException("Not supported in v1"); http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/abbdd6b4/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java ---------------------------------------------------------------------- diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java index 30872c9..82dfdd4 100644 --- a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java +++ b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java @@ -215,7 +215,7 @@ public class EntityManagerImpl implements EntityManager { @Override - public void init(EntityManagerFactory emf, EntityIndex entityIndex, UUID applicationId) { + public void init(EntityManagerFactory emf, UUID applicationId) { init( (EntityManagerFactoryImpl)emf, null, null, applicationId, false); } @@ -2908,10 +2908,6 @@ public class EntityManagerImpl implements EntityManager { throw new UnsupportedOperationException("Not supported."); } - @Override - public Health getIndexHealth() { - return Health.GREEN; // no good way to assess index status using old-school entity manager - } @Override http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/abbdd6b4/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java ---------------------------------------------------------------------- diff --git a/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java b/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java index e27420b..5c166c5 100644 --- a/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java +++ b/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java @@ -24,7 +24,6 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; -import org.apache.usergrid.persistence.index.*; import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -42,11 +41,14 @@ import org.apache.usergrid.persistence.Entity; import org.apache.usergrid.persistence.EntityManager; import org.apache.usergrid.persistence.EntityRef; import org.apache.usergrid.persistence.Results; -import org.apache.usergrid.persistence.collection.CollectionScope; import org.apache.usergrid.persistence.collection.EntityCollectionManager; import org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory; import org.apache.usergrid.persistence.core.scope.ApplicationScope; import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl; +import org.apache.usergrid.persistence.index.ApplicationEntityIndex; +import org.apache.usergrid.persistence.index.EntityIndexFactory; +import org.apache.usergrid.persistence.index.IndexScope; +import org.apache.usergrid.persistence.index.SearchTypes; import org.apache.usergrid.persistence.index.impl.IndexScopeImpl; import org.apache.usergrid.persistence.index.query.CandidateResults; import org.apache.usergrid.persistence.index.query.Query; @@ -58,7 +60,6 @@ import com.google.inject.Injector; import net.jcip.annotations.NotThreadSafe; import static org.apache.usergrid.corepersistence.CoreModule.EVENTS_DISABLED; -import static org.apache.usergrid.corepersistence.util.CpNamingUtils.getCollectionScopeNameFromEntityType; import static org.apache.usergrid.persistence.Schema.TYPE_APPLICATION; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -462,13 +463,11 @@ public class StaleIndexCleanupTest extends AbstractCoreIT { EntityManager em = app.getEntityManager(); - CollectionScope cs = getCollectionScopeNameFromEntityType( - new SimpleId( em.getApplicationId(), TYPE_APPLICATION ), eref.getType() ); EntityCollectionManagerFactory ecmf = SpringResource.getInstance().getBean( Injector.class ).getInstance( EntityCollectionManagerFactory.class ); - EntityCollectionManager ecm = ecmf.createCollectionManager( cs ); + EntityCollectionManager ecm = ecmf.createCollectionManager( new ApplicationScopeImpl( new SimpleId(em.getApplicationId(), "application" ) ) ); return ecm.load( new SimpleId( eref.getUuid(), eref.getType() ) ) .toBlocking().lastOrDefault( null ); http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/abbdd6b4/stack/core/src/test/java/org/apache/usergrid/corepersistence/migration/EntityTypeMappingMigrationIT.java ---------------------------------------------------------------------- diff --git a/stack/core/src/test/java/org/apache/usergrid/corepersistence/migration/EntityTypeMappingMigrationIT.java b/stack/core/src/test/java/org/apache/usergrid/corepersistence/migration/EntityTypeMappingMigrationIT.java index 88f56c8..84215a1 100644 --- a/stack/core/src/test/java/org/apache/usergrid/corepersistence/migration/EntityTypeMappingMigrationIT.java +++ b/stack/core/src/test/java/org/apache/usergrid/corepersistence/migration/EntityTypeMappingMigrationIT.java @@ -25,11 +25,11 @@ import org.junit.Test; import org.apache.usergrid.AbstractCoreIT; import org.apache.usergrid.corepersistence.ManagerCache; import org.apache.usergrid.corepersistence.util.CpNamingUtils; -import org.apache.usergrid.persistence.collection.CollectionScope; -import org.apache.usergrid.persistence.collection.impl.CollectionScopeImpl; import org.apache.usergrid.persistence.collection.serialization.impl.migration.EntityIdScope; import org.apache.usergrid.persistence.core.migration.data.TestMigrationDataProvider; import org.apache.usergrid.persistence.core.migration.data.TestProgressObserver; +import org.apache.usergrid.persistence.core.scope.ApplicationScope; +import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl; import org.apache.usergrid.persistence.map.MapManager; import org.apache.usergrid.persistence.map.MapScope; import org.apache.usergrid.persistence.map.impl.MapScopeImpl; @@ -56,7 +56,7 @@ public class EntityTypeMappingMigrationIT { final Id applicationId = createId("application"); - final CollectionScope scope1 = new CollectionScopeImpl( applicationId, applicationId, "things" ); + final ApplicationScope scope1 = new ApplicationScopeImpl( applicationId ); final Id entityId1 = createId("thing"); @@ -66,9 +66,7 @@ public class EntityTypeMappingMigrationIT { - final Id subCollectionOwner = createId("owners"); - - final CollectionScope scope2 = new CollectionScopeImpl( applicationId, subCollectionOwner, "foos" ); + final ApplicationScope scope2 = new ApplicationScopeImpl( applicationId); final Id entityId2 = createId("foo"); http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/abbdd6b4/stack/rest/src/main/java/org/apache/usergrid/rest/RootResource.java ---------------------------------------------------------------------- diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/RootResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/RootResource.java index e4086f9..2a78fc6 100644 --- a/stack/rest/src/main/java/org/apache/usergrid/rest/RootResource.java +++ b/stack/rest/src/main/java/org/apache/usergrid/rest/RootResource.java @@ -186,8 +186,8 @@ public class RootResource extends AbstractContextResource implements MetricProce throw new RuntimeException("Error connecting to datastore"); } - EntityManager em = emf.getEntityManager( emf.getManagementAppId() ); - if ( em.getIndexHealth().equals( Health.RED) ) { + + if ( emf.getIndexHealth().equals( Health.RED) ) { throw new RuntimeException("Management app index is status RED"); } } @@ -205,7 +205,7 @@ public class RootResource extends AbstractContextResource implements MetricProce // Core Persistence Query Index module status for Management App Index EntityManager em = emf.getEntityManager( emf.getManagementAppId() ); - node.put( "managementAppIndexStatus", em.getIndexHealth().toString() ); + node.put( "managementAppIndexStatus", emf.getIndexHealth().toString() ); dumpMetrics( node ); response.setProperty( "status", node ); http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/abbdd6b4/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/applications/ApplicationResource.java ---------------------------------------------------------------------- diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/applications/ApplicationResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/applications/ApplicationResource.java index d7dcd87..eef5775 100644 --- a/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/applications/ApplicationResource.java +++ b/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/applications/ApplicationResource.java @@ -390,7 +390,7 @@ public class ApplicationResource extends AbstractContextResource { Map<String, Object> statusMap = new HashMap<String, Object>(); EntityManager em = emf.getEntityManager( applicationId ); - if ( !em.getIndexHealth().equals( Health.RED ) ) { + if ( !emf.getIndexHealth().equals( Health.RED ) ) { statusMap.put("message", "Index Health Status RED for application " + applicationId ); return Response.status( SC_INTERNAL_SERVER_ERROR ).entity( statusMap ).build(); }
