Another cherry pick with another issue with collection resource it and nothing else.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/aa7f776e Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/aa7f776e Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/aa7f776e Branch: refs/heads/release-2.1.1 Commit: aa7f776edae7f9a2cb4c1577b6c86e1c028f082a Parents: 99c75cc Author: George Reyes <[email protected]> Authored: Fri Mar 11 15:19:52 2016 -0800 Committer: George Reyes <[email protected]> Committed: Thu Mar 24 09:08:05 2016 -0700 ---------------------------------------------------------------------- .../corepersistence/CpEntityManager.java | 11 --- .../corepersistence/CpRelationManager.java | 54 ++++++++++++ .../asyncevents/EventBuilderImpl.java | 2 +- .../corepersistence/index/IndexServiceImpl.java | 73 ++++++++++++++-- .../impl/EntityCollectionManagerImpl.java | 2 +- .../index/impl/EsEntityIndexBatchImpl.java | 9 +- .../collection/CollectionsResourceIT.java | 87 +++++++++++++++++++- .../services/AbstractCollectionService.java | 1 - 8 files changed, 215 insertions(+), 24 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/usergrid/blob/aa7f776e/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java ---------------------------------------------------------------------- diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java index d5ba946..1fbacbe 100644 --- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java +++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java @@ -1770,17 +1770,6 @@ public class CpEntityManager implements EntityManager { return obj; } -// @Override -// public Entity createCollectionSchema( String collectionName ){ -// MapManager mm = getMapManagerForTypes(); -// String jsonMap = mm.getString( collectionName ); -// -// Object obj = JsonUtils.parse( jsonMap ); -// -// return null; -// } - - @Override public void grantRolePermission( String roleName, String permission ) throws Exception { roleName = roleName.toLowerCase(); http://git-wip-us.apache.org/repos/asf/usergrid/blob/aa7f776e/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java ---------------------------------------------------------------------- diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java index cfb8dfa..faffbba 100644 --- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java +++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java @@ -18,6 +18,7 @@ package org.apache.usergrid.corepersistence; import java.util.ArrayList; +import java.util.Collection; import java.util.HashSet; import java.util.Iterator; import java.util.LinkedHashMap; @@ -69,10 +70,15 @@ import org.apache.usergrid.persistence.index.EntityIndex; import org.apache.usergrid.persistence.index.EntityIndexBatch; import org.apache.usergrid.persistence.index.SearchEdge; import org.apache.usergrid.persistence.index.query.Identifier; +import org.apache.usergrid.persistence.map.MapManager; +import org.apache.usergrid.persistence.map.MapScope; import org.apache.usergrid.persistence.model.entity.Id; import org.apache.usergrid.persistence.model.entity.SimpleId; +import org.apache.usergrid.persistence.model.field.Field; +import org.apache.usergrid.persistence.model.field.NullField; import org.apache.usergrid.persistence.schema.CollectionInfo; import org.apache.usergrid.utils.InflectionUtils; +import org.apache.usergrid.utils.JsonUtils; import org.apache.usergrid.utils.MapUtils; import com.google.common.base.Optional; @@ -90,6 +96,7 @@ import static org.apache.usergrid.persistence.Schema.COLLECTION_ROLES; import static org.apache.usergrid.persistence.Schema.PROPERTY_INACTIVITY; import static org.apache.usergrid.persistence.Schema.PROPERTY_NAME; import static org.apache.usergrid.persistence.Schema.PROPERTY_TITLE; +import static org.apache.usergrid.persistence.Schema.TYPE_APPLICATION; import static org.apache.usergrid.persistence.Schema.TYPE_ENTITY; import static org.apache.usergrid.persistence.Schema.TYPE_ROLE; import static org.apache.usergrid.persistence.Schema.getDefaultSchema; @@ -349,6 +356,15 @@ public class CpRelationManager implements RelationManager { @Override public Entity addToCollection( String collectionName, EntityRef itemRef ) throws Exception { + /** + * Get the map manager for uuid mapping + */ + Id mapOwner = new SimpleId( applicationId, TYPE_APPLICATION ); + + final MapScope ms = CpNamingUtils.getEntityTypeMapScope(mapOwner ); + + MapManager mm = managerCache.getMapManager( ms ); + Preconditions.checkNotNull( itemRef, "itemref is null" ); CollectionInfo collection = getDefaultSchema().getCollection( headEntity.getType(), collectionName ); if ( ( collection != null && collection.getType() != null ) && !collection.getType() @@ -392,6 +408,44 @@ public class CpRelationManager implements RelationManager { GraphManager gm = managerCache.getGraphManager( applicationScope ); +// //do logic here +// String jsonMap = mm.getString( collectionName ); +// +// Set<String> defaultProperties = null; +// +// if(jsonMap != null) { +// +// Map jsonMapData = ( Map ) JsonUtils.parse( jsonMap ); +// Schema schema = Schema.getDefaultSchema(); +// defaultProperties = schema.getRequiredProperties( collectionName ); +// //TODO: additional logic to +// ArrayList fieldsToKeep = ( ArrayList ) jsonMapData.get( "fields" ); +// defaultProperties.addAll( fieldsToKeep ); +// +// } +// +// Collection<String> trimmedFields = null; +// if(defaultProperties!=null){ +// // if(cpHeadEntity.getFields()) +// final Set<String> finalDefaultProperties = defaultProperties; +// trimmedFields = memberEntity.getFieldMap().keySet(); +// Iterator collectionIterator = trimmedFields.iterator(); +// while(collectionIterator.hasNext()){ +// if(!finalDefaultProperties.contains( collectionIterator.next() )){ +// collectionIterator.remove(); +// } +// } + + +// trimmedFields.forEach( element -> { +// if(!finalDefaultProperties.contains( element )){ +// memberEntity.removeField( element ); +// } +// }); + +// } + + gm.writeEdge( edge ).doOnNext( writtenEdge -> { if ( logger.isDebugEnabled() ) { logger.debug( "Wrote edge {}", writtenEdge ); http://git-wip-us.apache.org/repos/asf/usergrid/blob/aa7f776e/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java ---------------------------------------------------------------------- diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java index 2edc668..f2a8ff3 100644 --- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java +++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java @@ -141,7 +141,7 @@ public class EventBuilderImpl implements EventBuilder { return new EntityDeleteResults( deIndexObservable, ecmDeleteObservable, graphCompactObservable ); } - + //TODO: have the map manager here and filter out the entities that are going to be loaded. @Override public Observable<IndexOperationMessage> buildEntityIndex( final EntityIndexOperation entityIndexOperation ) { http://git-wip-us.apache.org/repos/asf/usergrid/blob/aa7f776e/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java ---------------------------------------------------------------------- diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java index 6c07212..01bc516 100644 --- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java +++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java @@ -20,11 +20,22 @@ package org.apache.usergrid.corepersistence.index; +import java.util.ArrayList; +import java.util.Collection; import java.util.Iterator; +import java.util.Map; +import java.util.Set; import java.util.UUID; import java.util.concurrent.TimeUnit; +import org.apache.usergrid.persistence.Schema; import org.apache.usergrid.persistence.index.*; +import org.apache.usergrid.persistence.map.MapManager; +import org.apache.usergrid.persistence.map.MapManagerFactory; +import org.apache.usergrid.persistence.map.MapScope; +import org.apache.usergrid.persistence.model.entity.SimpleId; +import org.apache.usergrid.persistence.model.field.Field; +import org.apache.usergrid.utils.JsonUtils; import org.apache.usergrid.utils.UUIDUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -52,6 +63,7 @@ import rx.Observable; import static org.apache.usergrid.corepersistence.util.CpNamingUtils.createSearchEdgeFromSource; import static org.apache.usergrid.corepersistence.util.CpNamingUtils.generateScopeFromSource; import static org.apache.usergrid.corepersistence.util.CpNamingUtils.generateScopeFromTarget; +import static org.apache.usergrid.persistence.Schema.TYPE_APPLICATION; import static org.apache.usergrid.persistence.Schema.getDefaultSchema; @@ -66,6 +78,7 @@ public class IndexServiceImpl implements IndexService { private final GraphManagerFactory graphManagerFactory; private final EntityIndexFactory entityIndexFactory; + private final MapManagerFactory mapManagerFactory; private final EdgesObservable edgesObservable; private final IndexFig indexFig; private final IndexLocationStrategyFactory indexLocationStrategyFactory; @@ -75,9 +88,13 @@ public class IndexServiceImpl implements IndexService { @Inject public IndexServiceImpl( final GraphManagerFactory graphManagerFactory, final EntityIndexFactory entityIndexFactory, - final EdgesObservable edgesObservable, final IndexFig indexFig, final IndexLocationStrategyFactory indexLocationStrategyFactory, final MetricsFactory metricsFactory ) { + final MapManagerFactory mapManagerFactory, + final EdgesObservable edgesObservable, final IndexFig indexFig, + final IndexLocationStrategyFactory indexLocationStrategyFactory, + final MetricsFactory metricsFactory ) { this.graphManagerFactory = graphManagerFactory; this.entityIndexFactory = entityIndexFactory; + this.mapManagerFactory = mapManagerFactory; this.edgesObservable = edgesObservable; this.indexFig = indexFig; this.indexLocationStrategyFactory = indexLocationStrategyFactory; @@ -93,7 +110,6 @@ public class IndexServiceImpl implements IndexService { final GraphManager gm = graphManagerFactory.createEdgeManager( applicationScope ); final EntityIndex ei = entityIndexFactory.createEntityIndex(indexLocationStrategyFactory.getIndexLocationStrategy(applicationScope)); - final Id entityId = entity.getId(); @@ -142,11 +158,58 @@ public class IndexServiceImpl implements IndexService { final EntityIndexBatch batch = ei.createBatch(); - if (logger.isDebugEnabled()) { + // if (logger.isDebugEnabled()) { logger.debug("adding edge {} to batch for entity {}", indexEdge, entity); - } + // } + + + indexEdge.getNodeId().getUuid(); + + //System.out.println("hello"); + + Id mapOwner = new SimpleId( indexEdge.getNodeId().getUuid(), TYPE_APPLICATION ); + + final MapScope ms = CpNamingUtils.getEntityTypeMapScope(mapOwner ); + + MapManager mm = mapManagerFactory.createMapManager( ms ); + + + + String jsonMap = mm.getString( indexEdge.getEdgeName().split( "\\|" )[1] ); + + Set<String> defaultProperties = null; + + if(jsonMap != null) { + + Map jsonMapData = ( Map ) JsonUtils.parse( jsonMap ); + Schema schema = Schema.getDefaultSchema(); + defaultProperties = schema.getRequiredProperties( indexEdge.getEdgeName().split( "\\|" )[1]); + //TODO: additional logic to + ArrayList fieldsToKeep = ( ArrayList ) jsonMapData.get( "fields" ); + defaultProperties.addAll( fieldsToKeep ); + + } + + Entity filteredEntity = new Entity( entity.getId(),entity.getVersion() ); + filteredEntity.setFieldMap( entity.getFieldMap() ); + + Collection<String> trimmedFields = null; + if(defaultProperties!=null) { + // if(cpHeadEntity.getFields()) + final Set<String> finalDefaultProperties = defaultProperties; + trimmedFields = entity.getFieldMap().keySet(); + Iterator collectionIterator = trimmedFields.iterator(); + while ( collectionIterator.hasNext() ) { + String fieldName = ( String ) collectionIterator.next(); + if ( !finalDefaultProperties.contains( fieldName ) ) { + //collectionIterator.remove(); + filteredEntity.removeField( fieldName ); + } + } + } + - batch.index( indexEdge, entity ); + batch.index( indexEdge, filteredEntity ); return batch.build(); } ); http://git-wip-us.apache.org/repos/asf/usergrid/blob/aa7f776e/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java index 06025d3..e71e6bb 100644 --- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java +++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java @@ -184,7 +184,7 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager { // create our observable and start the write final CollectionIoEvent<Entity> writeData = new CollectionIoEvent<Entity>( applicationScope, entity ); - Observable<CollectionIoEvent<MvccEntity>> observable = stageRunner( writeData, writeStart ); + Observable<CollectionIoEvent<MvccEntity>> observable = stageRunner( writeData, writeStart ); final Observable<Entity> write = observable.map( writeCommit ) http://git-wip-us.apache.org/repos/asf/usergrid/blob/aa7f776e/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java index 9427be1..f703e32 100644 --- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java +++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java @@ -28,6 +28,7 @@ import org.slf4j.LoggerFactory; import org.apache.usergrid.persistence.core.scope.ApplicationScope; import org.apache.usergrid.persistence.core.util.ValidationUtils; import org.apache.usergrid.persistence.index.utils.IndexValidationUtils; +import org.apache.usergrid.persistence.map.MapManager; import org.apache.usergrid.persistence.model.entity.Entity; import org.apache.usergrid.persistence.model.entity.Id; @@ -68,10 +69,10 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch { final String writeAlias = alias.getWriteAlias(); - if ( logger.isDebugEnabled() ) { - logger.debug( "Indexing to alias {} with scope {} on edge {} with entity data {}", - writeAlias, applicationScope, indexEdge, entity ); - } + //if ( logger.isDebugEnabled() ) { + logger.info( "Indexing to alias {} with scope {} on edge {} with entity data {}", + writeAlias, applicationScope, indexEdge, entity.getFieldMap().keySet() ); + //} //add app id for indexing container.addIndexRequest(new IndexOperation(writeAlias, applicationScope, indexEdge, entity)); http://git-wip-us.apache.org/repos/asf/usergrid/blob/aa7f776e/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/CollectionsResourceIT.java ---------------------------------------------------------------------- diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/CollectionsResourceIT.java b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/CollectionsResourceIT.java index cea9d5e..1aa91d1 100644 --- a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/CollectionsResourceIT.java +++ b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/CollectionsResourceIT.java @@ -21,12 +21,14 @@ import com.fasterxml.jackson.databind.JsonNode; import org.apache.usergrid.persistence.Schema; import org.apache.usergrid.persistence.entities.Application; + import org.apache.usergrid.rest.test.resource.AbstractRestIT; import org.apache.usergrid.rest.test.resource.model.ApiResponse; import org.apache.usergrid.rest.test.resource.model.Collection; import org.apache.usergrid.rest.test.resource.model.Entity; import org.apache.usergrid.rest.test.resource.model.QueryParameters; import org.apache.usergrid.rest.test.resource.model.Token; + import org.junit.Ignore; import org.junit.Test; import org.slf4j.Logger; @@ -123,7 +125,7 @@ public class CollectionsResourceIT extends AbstractRestIT { * @throws Exception */ @Test - public void postToReservedField() throws Exception { + public void postToCollectionSchemaUpdateExistingCollection() throws Exception { //Create test collection with test entity that is full text indexed. Entity testEntity = new Entity(); @@ -163,6 +165,7 @@ public class CollectionsResourceIT extends AbstractRestIT { assertEquals( ( thing ).get( "lastUpdateBy" ),testCollectionSchema.get( "lastUpdateBy" ) ); assertEquals( ( thing ).get( "lastReindexed" ),testCollectionSchema.get( "lastReindexed" ) ); + //TODO: this test doesn't check to see if create checks the schema. Only that the reindex removes whats already there. ArrayList<String> schema = ( ArrayList<String> ) testCollectionSchema.get( "fields" ); assertEquals( "one",schema.get( 0 ) ); @@ -179,6 +182,88 @@ public class CollectionsResourceIT extends AbstractRestIT { } /** + * Create test collection + * Give collection an indexing schema + * Give collection a new entity and ensure it only indexes wht is in the schema + * Reindex and make sure old entity with full text indexing is reindexed with the schema. + * + * @throws Exception + */ + @Test + public void postToCollectionSchemaWithSchemaFirst() throws Exception { + + + + //Creating schema. + //this could be changed to a hashmap. + ArrayList<String> indexingArray = new ArrayList<>( ); + indexingArray.add( "one" ); + indexingArray.add( "name" ); + + //TODO: add indexing array to the backend/test once you finish the regular selective indexing. + //indexingArray.add( "field.three.index.array" ); + + //field "fields" is required. + Entity payload = new Entity(); + payload.put( "fields", indexingArray); + + //Post index to the collection metadata + // Entity thing = this.app().collection( "testCollection" ).collection( "_indexes" ).post( payload ); + refreshIndex(); + + //Create test collection with test entity that is full text indexed. + Entity testEntity = new Entity(); + //testEntity.put( "name", "tester"); + testEntity.put( "one", "helper" ); + //this field shouldn't persist after reindexing. + testEntity.put( "two","query" ); + + //TODO: add arrays to the indexing test + //testEntity.put("array","array stuff here"); + + Entity returnedEntity = this.app().collection( "testCollection" ).post( testEntity ); + refreshIndex(); + //testEntity.put( "three","notthree" ); + //ApiResponse updatedEntity = this.app().collection( "testCollection" ).put( null,testEntity ); + + //Below is what needs to be implemented along with the index call above + +// //Get the collection schema and verify that it contains the same schema as posted above. +// Collection collection = this.app().collection( "testCollection" ).collection( "_index" ).get(); +// +// LinkedHashMap testCollectionSchema = (LinkedHashMap)collection.getResponse().getData(); +// //TODO: the below will have to be replaced by the values that I deem correct. +// assertEquals( ( thing ).get( "lastUpdated" ), testCollectionSchema.get( "lastUpdated" )); +// assertEquals( ( thing ).get( "lastUpdateBy" ),testCollectionSchema.get( "lastUpdateBy" ) ); +// assertEquals( ( thing ).get( "lastReindexed" ),testCollectionSchema.get( "lastReindexed" ) ); + +// //TODO: this test doesn't check to see if create checks the schema. Only that the reindex removes whats already there. +// ArrayList<String> schema = ( ArrayList<String> ) testCollectionSchema.get( "fields" ); +// assertEquals( "one",schema.get( 0 ) ); + + //Reindex and verify that the entity only has field one index. +// this.app().collection( "testCollection" ).collection( "_reindex" ).post(); +// +// refreshIndex(); + + //this needs to be a query. not a graph. This should still return the full entity. + + String query = "two ='query'"; + QueryParameters queryParameters = new QueryParameters().setQuery(query); + +//having a name breaks it. Need to get rid of the stack trace and also + Collection tempEntity = this.app().collection( "testCollection" ).get(queryParameters,true); + + + Entity reindexedEntity = tempEntity.getResponse().getEntity(); //this.app().collection( "testCollection" ).entity( returnedEntity.getUuid() ).get(); + assertEquals( "helper",reindexedEntity.get( "one" ) ); + assertNull( reindexedEntity.get( "two" ) ); + //not sure if this should have some kind of sleep here because this reindex will be heavily throttled. + + } + + + /** * Test posts with a user level token on a path with permissions */ //TODO: App level permissions aren't functioning. http://git-wip-us.apache.org/repos/asf/usergrid/blob/aa7f776e/stack/services/src/main/java/org/apache/usergrid/services/AbstractCollectionService.java ---------------------------------------------------------------------- diff --git a/stack/services/src/main/java/org/apache/usergrid/services/AbstractCollectionService.java b/stack/services/src/main/java/org/apache/usergrid/services/AbstractCollectionService.java index 4d73fa5..8b23e7a 100644 --- a/stack/services/src/main/java/org/apache/usergrid/services/AbstractCollectionService.java +++ b/stack/services/src/main/java/org/apache/usergrid/services/AbstractCollectionService.java @@ -333,7 +333,6 @@ public class AbstractCollectionService extends AbstractService { setServiceManager( serviceRequest.getServices() ); ServiceContext context = serviceRequest.getAppContext(); - context.setAction( ServiceAction.POST ); checkPermissionsForCollection( context ); Subject currentUser = SubjectUtils.getSubject(); Object currentUser2 =currentUser.getPrincipal();
