no message
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/f0480b48 Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/f0480b48 Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/f0480b48 Branch: refs/heads/release-2.1.1 Commit: f0480b4881a097e50ad2153e735268834b40dbb9 Parents: a2ec1df Author: George Reyes <[email protected]> Authored: Wed Mar 16 11:38:10 2016 -0700 Committer: George Reyes <[email protected]> Committed: Thu Mar 24 09:14:15 2016 -0700 ---------------------------------------------------------------------- .../corepersistence/index/IndexServiceImpl.java | 101 +++++++++++-------- .../persistence/index/EntityIndexBatch.java | 3 + .../index/impl/EntityToMapConverter.java | 7 ++ .../index/impl/EsEntityIndexBatchImpl.java | 27 ++++- .../persistence/index/impl/IndexOperation.java | 19 +++- .../collection/CollectionsResourceIT.java | 101 ++++++++++++++++++- 6 files changed, 213 insertions(+), 45 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/usergrid/blob/f0480b48/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java ---------------------------------------------------------------------- diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java index 01bc516..9e0e799 100644 --- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java +++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java @@ -22,6 +22,7 @@ package org.apache.usergrid.corepersistence.index; import java.util.ArrayList; import java.util.Collection; +import java.util.HashSet; import java.util.Iterator; import java.util.Map; import java.util.Set; @@ -30,6 +31,8 @@ import java.util.concurrent.TimeUnit; import org.apache.usergrid.persistence.Schema; import org.apache.usergrid.persistence.index.*; +import org.apache.usergrid.persistence.index.impl.EntityField; +import org.apache.usergrid.persistence.index.impl.IndexOperation; import org.apache.usergrid.persistence.map.MapManager; import org.apache.usergrid.persistence.map.MapManagerFactory; import org.apache.usergrid.persistence.map.MapScope; @@ -158,65 +161,83 @@ public class IndexServiceImpl implements IndexService { final EntityIndexBatch batch = ei.createBatch(); - // if (logger.isDebugEnabled()) { + if (logger.isDebugEnabled()) { logger.debug("adding edge {} to batch for entity {}", indexEdge, entity); - // } + } + Map<String, Object> map = getFilteredStringObjectMap( applicationScope, entity, indexEdge ); - indexEdge.getNodeId().getUuid(); + if(map!=null){ + batch.index( indexEdge, entity ,map); + } + else{ + batch.index( indexEdge,entity ); + } - //System.out.println("hello"); - Id mapOwner = new SimpleId( indexEdge.getNodeId().getUuid(), TYPE_APPLICATION ); + return batch.build(); + } ); - final MapScope ms = CpNamingUtils.getEntityTypeMapScope(mapOwner ); + return ObservableTimer.time( batches, addTimer ); - MapManager mm = mapManagerFactory.createMapManager( ms ); + } - String jsonMap = mm.getString( indexEdge.getEdgeName().split( "\\|" )[1] ); + private Map<String, Object> getFilteredStringObjectMap( final ApplicationScope applicationScope, final Entity entity, + final IndexEdge indexEdge ) {IndexOperation indexOperation = new IndexOperation( ); - Set<String> defaultProperties = null; - if(jsonMap != null) { + indexEdge.getNodeId().getUuid(); - Map jsonMapData = ( Map ) JsonUtils.parse( jsonMap ); - Schema schema = Schema.getDefaultSchema(); - defaultProperties = schema.getRequiredProperties( indexEdge.getEdgeName().split( "\\|" )[1]); - //TODO: additional logic to - ArrayList fieldsToKeep = ( ArrayList ) jsonMapData.get( "fields" ); - defaultProperties.addAll( fieldsToKeep ); + Id mapOwner = new SimpleId( indexEdge.getNodeId().getUuid(), TYPE_APPLICATION ); - } + final MapScope ms = CpNamingUtils.getEntityTypeMapScope(mapOwner ); - Entity filteredEntity = new Entity( entity.getId(),entity.getVersion() ); - filteredEntity.setFieldMap( entity.getFieldMap() ); - - Collection<String> trimmedFields = null; - if(defaultProperties!=null) { - // if(cpHeadEntity.getFields()) - final Set<String> finalDefaultProperties = defaultProperties; - trimmedFields = entity.getFieldMap().keySet(); - Iterator collectionIterator = trimmedFields.iterator(); - while ( collectionIterator.hasNext() ) { - String fieldName = ( String ) collectionIterator.next(); - if ( !finalDefaultProperties.contains( fieldName ) ) { - //collectionIterator.remove(); - filteredEntity.removeField( fieldName ); - } - } - } + MapManager mm = mapManagerFactory.createMapManager( ms ); + String jsonMap = mm.getString( indexEdge.getEdgeName().split( "\\|" )[1] ); - batch.index( indexEdge, filteredEntity ); - - return batch.build(); - } ); - - return ObservableTimer.time( batches, addTimer ); + Set<String> defaultProperties = null; + if(jsonMap != null) { + Map jsonMapData = ( Map ) JsonUtils.parse( jsonMap ); + Schema schema = Schema.getDefaultSchema(); + defaultProperties = schema.getRequiredProperties( indexEdge.getEdgeName().split( "\\|" )[1]); + //TODO: additional logic to + ArrayList fieldsToKeep = ( ArrayList ) jsonMapData.get( "fields" ); + defaultProperties.addAll( fieldsToKeep ); + } + else + return null; + + Map<String, Object> map = indexOperation.convertedEntityToBeIndexed( applicationScope,indexEdge,entity ); + HashSet mapFields = ( HashSet ) map.get( "fields" ); + + if(defaultProperties!=null) { + final Set<String> finalDefaultProperties = defaultProperties; + Iterator collectionIterator = mapFields.iterator(); + while ( collectionIterator.hasNext() ) { + EntityField testedField = ( EntityField ) collectionIterator.next(); + String fieldName = ( String ) (testedField).get( "name" ); + //TODO: need to change the logic here such that default properties does a check against each every field. + //it is very likely that the below does O(n) comparisons anyways so doing it for each value in a loop + //would be the right equivalent here. Even if it isn't very efficient. + //maybe if it was a hashmap, and if the hashmap didn't contain the value then we remove the field. + //but now for each hash value we have to do a check to see it doesn't start with a certain value. + //making this an O(n) operation anyways. + + //so for each value in the finalDefaultProperties check to see if it matches the current property + //then if it doesn't do a split of the properties and see if any of them match for the split + //if they do match then allow in, otherwise don't allow. + + if ( !finalDefaultProperties.contains( fieldName ) ) { + mapFields.remove( testedField ); + } + } + } + return map; } http://git-wip-us.apache.org/repos/asf/usergrid/blob/f0480b48/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndexBatch.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndexBatch.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndexBatch.java index 17dd4d3..e3a40f6 100644 --- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndexBatch.java +++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndexBatch.java @@ -19,6 +19,7 @@ package org.apache.usergrid.persistence.index;/* import java.util.List; +import java.util.Map; import java.util.UUID; import org.apache.usergrid.persistence.index.impl.IndexOperationMessage; @@ -53,6 +54,8 @@ public interface EntityIndexBatch { */ EntityIndexBatch deindex( final SearchEdge searchEdge, final CandidateResult result ); + EntityIndexBatch index( IndexEdge indexEdge, Entity entity, Map flattenedEntityMap ); + /** * Remove index of entity. * http://git-wip-us.apache.org/repos/asf/usergrid/blob/f0480b48/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EntityToMapConverter.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EntityToMapConverter.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EntityToMapConverter.java index d0a5c44..77a8d92 100644 --- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EntityToMapConverter.java +++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EntityToMapConverter.java @@ -17,16 +17,22 @@ package org.apache.usergrid.persistence.index.impl; +import java.util.ArrayList; +import java.util.Collection; import java.util.HashMap; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Set; import org.apache.usergrid.persistence.core.scope.ApplicationScope; import org.apache.usergrid.persistence.index.IndexEdge; +import org.apache.usergrid.persistence.map.MapManager; +import org.apache.usergrid.persistence.map.MapScope; import org.apache.usergrid.persistence.model.entity.Entity; import org.apache.usergrid.persistence.model.entity.EntityMap; import org.apache.usergrid.persistence.model.entity.Id; +import org.apache.usergrid.persistence.model.entity.SimpleId; import static org.apache.usergrid.persistence.index.impl.IndexingUtils.APPLICATION_ID_FIELDNAME; import static org.apache.usergrid.persistence.index.impl.IndexingUtils.EDGE_NAME_FIELDNAME; @@ -45,6 +51,7 @@ import static org.apache.usergrid.persistence.index.impl.IndexingUtils.getType; import static org.apache.usergrid.persistence.index.impl.IndexingUtils.nodeId; + /** * Convert a CP entity to an elasticsearch document */ http://git-wip-us.apache.org/repos/asf/usergrid/blob/f0480b48/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java index f703e32..5296978 100644 --- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java +++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java @@ -19,6 +19,7 @@ package org.apache.usergrid.persistence.index.impl; +import java.util.Map; import java.util.UUID; import org.apache.usergrid.persistence.index.*; @@ -31,6 +32,7 @@ import org.apache.usergrid.persistence.index.utils.IndexValidationUtils; import org.apache.usergrid.persistence.map.MapManager; import org.apache.usergrid.persistence.model.entity.Entity; import org.apache.usergrid.persistence.model.entity.Id; +import org.apache.usergrid.persistence.model.entity.SimpleId; public class EsEntityIndexBatchImpl implements EntityIndexBatch { @@ -69,16 +71,35 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch { final String writeAlias = alias.getWriteAlias(); - //if ( logger.isDebugEnabled() ) { - logger.info( "Indexing to alias {} with scope {} on edge {} with entity data {}", + if ( logger.isDebugEnabled() ) { + logger.debug( "Indexing to alias {} with scope {} on edge {} with entity data {}", writeAlias, applicationScope, indexEdge, entity.getFieldMap().keySet() ); - //} + } //add app id for indexing container.addIndexRequest(new IndexOperation(writeAlias, applicationScope, indexEdge, entity)); return this; } + @Override + public EntityIndexBatch index ( final IndexEdge indexEdge, final Entity entity ,final Map flattenedEntityMap){ + IndexValidationUtils.validateIndexEdge(indexEdge); + ValidationUtils.verifyEntityWrite(entity); + ValidationUtils.verifyVersion( entity.getVersion() ); + + final String writeAlias = alias.getWriteAlias(); + + if ( logger.isDebugEnabled() ) { + logger.debug( "Indexing to alias {} with scope {} on edge {} with entity data {}", + writeAlias, applicationScope, indexEdge, flattenedEntityMap ); + } + + //add app id for indexing + container.addIndexRequest(new IndexOperation(writeAlias, applicationScope, + indexEdge,entity.getId(),entity.getVersion(),flattenedEntityMap) ); + return this; + } + @Override public EntityIndexBatch deindex( final SearchEdge searchEdge, final Id id, final UUID version ) { http://git-wip-us.apache.org/repos/asf/usergrid/blob/f0480b48/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexOperation.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexOperation.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexOperation.java index 28f2e0d..b51d0c7 100644 --- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexOperation.java +++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexOperation.java @@ -21,10 +21,12 @@ package org.apache.usergrid.persistence.index.impl; import java.util.Map; +import java.util.UUID; import org.apache.usergrid.persistence.core.scope.ApplicationScope; import org.apache.usergrid.persistence.index.IndexEdge; import org.apache.usergrid.persistence.model.entity.Entity; +import org.apache.usergrid.persistence.model.entity.Id; import org.elasticsearch.action.bulk.BulkRequestBuilder; import org.elasticsearch.action.index.IndexRequestBuilder; @@ -48,10 +50,21 @@ public class IndexOperation implements BatchOperation { public IndexOperation( final String writeAlias, final ApplicationScope applicationScope, IndexEdge indexEdge, Entity entity ) { - this(writeAlias,IndexingUtils.createIndexDocId(applicationScope, entity,indexEdge), EntityToMapConverter.convert(applicationScope,indexEdge, entity)); + + + this(writeAlias,IndexingUtils.createIndexDocId(applicationScope, entity,indexEdge) + ,EntityToMapConverter.convert(applicationScope,indexEdge, entity)); + } + public IndexOperation( final String writeAlias, final ApplicationScope applicationScope, IndexEdge indexEdge, + Id entityId, UUID version, Map<String,Object> data ) { + + this(writeAlias,IndexingUtils.createIndexDocId(applicationScope, entityId,version,indexEdge) + ,data); + } + public IndexOperation( final String writeAlias, String documentId, Map<String, Object> data ) { this.writeAlias = writeAlias; this.data = data; @@ -107,4 +120,8 @@ public class IndexOperation implements BatchOperation { result = 31 * result + data.hashCode(); return result; } + + public Map convertedEntityToBeIndexed(ApplicationScope applicationScope,IndexEdge indexEdge,Entity entity){ + return EntityToMapConverter.convert(applicationScope,indexEdge, entity); + } } http://git-wip-us.apache.org/repos/asf/usergrid/blob/f0480b48/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/CollectionsResourceIT.java ---------------------------------------------------------------------- diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/CollectionsResourceIT.java b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/CollectionsResourceIT.java index 2e4a3ff..ab46681 100644 --- a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/CollectionsResourceIT.java +++ b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/CollectionsResourceIT.java @@ -42,7 +42,6 @@ import java.util.Map; import java.util.Set; import java.util.ArrayList; - import static org.junit.Assert.*; @@ -230,6 +229,106 @@ public class CollectionsResourceIT extends AbstractRestIT { } + @Test + public void postToCollectionArraySchemaWithSchemaFirst() throws Exception { + + //Include the property labeled two to be index. + ArrayList<String> indexingArray = new ArrayList<>( ); + indexingArray.add( "two" ); + indexingArray.add( "one.key" ); + + //field "fields" is required. + Entity payload = new Entity(); + payload.put( "fields", indexingArray); + + //Post index to the collection metadata + this.app().collection( "testCollection" ).collection( "_indexes" ).post( payload ); + refreshIndex(); + + Map<String,Object> arrayFieldsForTesting = new HashMap<>(); + + arrayFieldsForTesting.put( "key","value" ); + arrayFieldsForTesting.put( "anotherKey","value2"); + + //Create test collection with a test entity that is partially indexed. + Entity testEntity = new Entity(); + testEntity.put( "one", arrayFieldsForTesting ); + testEntity.put( "two","query" ); + + //Post entity. + this.app().collection( "testCollection" ).post( testEntity ); + refreshIndex(); + + //Do a query to see if you can find the indexed query. + String query = "one.key = 'value'"; + QueryParameters queryParameters = new QueryParameters().setQuery(query); + + //having a name breaks it. Need to get rid of the stack trace and also + Collection tempEntity = this.app().collection( "testCollection" ).get(queryParameters,true); + Entity reindexedEntity = tempEntity.getResponse().getEntity(); + assertEquals( "value2",((Map)reindexedEntity.get( "one" )).get( "anotherKey" ) ); + + //Verify if you can query on an entity that was not indexed and that no entities are returned. + query = "one.anotherKey = 'value2'"; + queryParameters = new QueryParameters().setQuery(query); + tempEntity = this.app().collection( "testCollection" ).get(queryParameters,true); + assertEquals(0,tempEntity.getResponse().getEntities().size()); + + } + + //test to reflect if one would index whatever is prefixed with that. Will + //need to do extensive testing to ensure that it will work with different instances. + @Test + public void postToCollectionSchemaArrayWithTopLevelIndexing() throws Exception { + + //Include the property labeled two to be index. + ArrayList<String> indexingArray = new ArrayList<>( ); + indexingArray.add( "two" ); + //this should work such that one.key and one.anotherKey should work. + indexingArray.add( "one" ); + + //field "fields" is required. + Entity payload = new Entity(); + payload.put( "fields", indexingArray); + + //Post index to the collection metadata + this.app().collection( "testCollection" ).collection( "_indexes" ).post( payload ); + refreshIndex(); + + Map<String,Object> arrayFieldsForTesting = new HashMap<>(); + + arrayFieldsForTesting.put( "key","value" ); + arrayFieldsForTesting.put( "anotherKey","value2"); + + //Create test collection with a test entity that is partially indexed. + Entity testEntity = new Entity(); + testEntity.put( "one", arrayFieldsForTesting ); + testEntity.put( "two","query" ); + + //Post entity. + this.app().collection( "testCollection" ).post( testEntity ); + refreshIndex(); + + //Do a query to see if you can find the indexed query. + String query = "one.key = 'value'"; + QueryParameters queryParameters = new QueryParameters().setQuery(query); + + //having a name breaks it. Need to get rid of the stack trace and also + Collection tempEntity = this.app().collection( "testCollection" ).get(queryParameters,true); + Entity reindexedEntity = tempEntity.getResponse().getEntity(); + assertEquals( "value2",((Map)reindexedEntity.get( "one" )).get( "anotherKey" ) ); + + //Verify if you can query on an entity that was not indexed and that no entities are returned. + //TODO: check that the below gets indexed as well. although the above should prove that at least one thing is getting indexed. +// query = "one.anotherKey = 'value2'"; +// queryParameters = new QueryParameters().setQuery(query); +// tempEntity = this.app().collection( "testCollection" ).get(queryParameters,true); +// assertEquals(0,tempEntity.getResponse().getEntities().size()); + + } + + + /** * Test posts with a user level token on a path with permissions
