Add some more comments and clarity to selective indexing.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/9f3720e9 Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/9f3720e9 Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/9f3720e9 Branch: refs/heads/release-2.1.1 Commit: 9f3720e998edd6343d78ec1b5360f1ce244a4d49 Parents: e5ae2a5 Author: George Reyes <[email protected]> Authored: Mon Apr 4 10:55:44 2016 -0700 Committer: George Reyes <[email protected]> Committed: Mon Apr 4 10:55:44 2016 -0700 ---------------------------------------------------------------------- .../corepersistence/CpEntityManager.java | 36 ++++++++------------ .../index/ReIndexServiceImpl.java | 1 - .../core/consistency/TimeServiceImpl.java | 5 ++- .../index/impl/EntityToMapConverter.java | 23 +++++++------ 4 files changed, 31 insertions(+), 34 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/usergrid/blob/9f3720e9/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java ---------------------------------------------------------------------- diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java index 6bbf0a3..4ee497c 100644 --- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java +++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java @@ -18,14 +18,11 @@ package org.apache.usergrid.corepersistence; import java.nio.ByteBuffer; import java.time.Instant; -import java.time.ZoneOffset; -import java.time.ZonedDateTime; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.Comparator; -import java.util.Date; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; @@ -38,22 +35,15 @@ import java.util.TreeMap; import java.util.TreeSet; import java.util.UUID; - -import org.apache.usergrid.corepersistence.index.IndexSchemaCache; -import org.apache.usergrid.corepersistence.index.IndexSchemaCacheFactory; -import org.apache.usergrid.corepersistence.service.CollectionService; -import org.apache.usergrid.corepersistence.service.ConnectionService; -import org.apache.usergrid.persistence.index.EntityIndex; -import org.apache.usergrid.utils.*; -import org.apache.usergrid.utils.ClassUtils; -import org.apache.usergrid.utils.UUIDUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.util.Assert; -import org.apache.avro.generic.GenericData; - import org.apache.usergrid.corepersistence.asyncevents.AsyncEventService; +import org.apache.usergrid.corepersistence.index.IndexSchemaCache; +import org.apache.usergrid.corepersistence.index.IndexSchemaCacheFactory; +import org.apache.usergrid.corepersistence.service.CollectionService; +import org.apache.usergrid.corepersistence.service.ConnectionService; import org.apache.usergrid.corepersistence.util.CpEntityMapUtils; import org.apache.usergrid.corepersistence.util.CpNamingUtils; import org.apache.usergrid.persistence.AggregateCounter; @@ -97,6 +87,7 @@ import org.apache.usergrid.persistence.exceptions.UnexpectedEntityTypeException; import org.apache.usergrid.persistence.graph.GraphManager; import org.apache.usergrid.persistence.graph.GraphManagerFactory; import org.apache.usergrid.persistence.graph.SearchEdgeType; +import org.apache.usergrid.persistence.index.EntityIndex; import org.apache.usergrid.persistence.index.query.CounterResolution; import org.apache.usergrid.persistence.index.query.Identifier; import org.apache.usergrid.persistence.map.MapManager; @@ -106,6 +97,13 @@ import org.apache.usergrid.persistence.model.entity.SimpleId; import org.apache.usergrid.persistence.model.field.Field; import org.apache.usergrid.persistence.model.field.StringField; import org.apache.usergrid.persistence.model.util.UUIDGenerator; +import org.apache.usergrid.utils.ClassUtils; +import org.apache.usergrid.utils.CompositeUtils; +import org.apache.usergrid.utils.InflectionUtils; +import org.apache.usergrid.utils.Inflector; +import org.apache.usergrid.utils.JsonUtils; +import org.apache.usergrid.utils.StringUtils; +import org.apache.usergrid.utils.UUIDUtils; import com.codahale.metrics.Meter; import com.codahale.metrics.Timer; @@ -1751,17 +1749,14 @@ public class CpEntityManager implements EntityManager { @Override public Map createCollectionSchema( String collectionName, String owner ,Map<String, Object> properties ){ - //haven't decided which one I should base off of which, maybe from epoch to utc //TODO: change timeservice as below then use timeservice. - //TODO: only allow a single reindex in elasticsearch at a time. akka. Instant timeInstance = Instant.now(); Long epoch = timeInstance.toEpochMilli(); Map<String,Object> schemaMap = new HashMap<>( ); - schemaMap.put("lastUpdated",epoch); //this needs the method that can extract the user from the token no matter the token. //Possible values are app credentials, org credentials, or the user email(Admin tokens). @@ -1775,7 +1770,7 @@ public class CpEntityManager implements EntityManager { java.util.Optional<Map> collectionIndexingSchema = indexSchemaCache.getCollectionSchema( collectionName ); - //If we do have a schema then parse it and add it to a list of properties we want to keep.Otherwise return. + //If there is an existing schema then take the lastReindexed time and keep it around.Otherwise initialize to 0. if ( collectionIndexingSchema.isPresent() ) { Map jsonMapData = collectionIndexingSchema.get(); schemaMap.put( "lastReindexed", jsonMapData.get( "lastReindexed" ) ); @@ -1786,6 +1781,8 @@ public class CpEntityManager implements EntityManager { ArrayList<String> fieldProperties = ( ArrayList<String> ) properties.get( "fields" ); + //do a check to see if you have a * field. If you do have a * field then ignore all other fields + //and only accept the * field. if(fieldProperties.contains( "*" )){ ArrayList<String> wildCardArrayList = new ArrayList<>( ); wildCardArrayList.add( "*" ); @@ -1795,9 +1792,6 @@ public class CpEntityManager implements EntityManager { schemaMap.putAll( properties ); } - //do a check to see if you have a * field. If you do have a * field then ignore all other fields - //and only accept the * field. That logic goes below and in the put. - indexSchemaCache.putCollectionSchema( collectionName, JsonUtils.mapToJsonString( schemaMap ) ); return schemaMap; http://git-wip-us.apache.org/repos/asf/usergrid/blob/9f3720e9/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java ---------------------------------------------------------------------- diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java index 125222a..0aa0f60 100644 --- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java +++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java @@ -44,7 +44,6 @@ import org.apache.usergrid.persistence.map.MapManager; import org.apache.usergrid.persistence.map.MapManagerFactory; import org.apache.usergrid.persistence.map.MapScope; import org.apache.usergrid.persistence.map.impl.MapScopeImpl; -import org.apache.usergrid.persistence.model.entity.SimpleId; import org.apache.usergrid.persistence.model.util.UUIDGenerator; import org.apache.usergrid.utils.InflectionUtils; import org.apache.usergrid.utils.JsonUtils; http://git-wip-us.apache.org/repos/asf/usergrid/blob/9f3720e9/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/consistency/TimeServiceImpl.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/consistency/TimeServiceImpl.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/consistency/TimeServiceImpl.java index 00a58d3..99fe756 100644 --- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/consistency/TimeServiceImpl.java +++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/consistency/TimeServiceImpl.java @@ -20,6 +20,9 @@ package org.apache.usergrid.persistence.core.consistency; +import java.time.Instant; + + /** * Simple time service to get the current system time. */ @@ -28,6 +31,6 @@ public class TimeServiceImpl implements TimeService{ @Override public long getCurrentTime() { - return System.currentTimeMillis(); + return Instant.now().toEpochMilli(); } } http://git-wip-us.apache.org/repos/asf/usergrid/blob/9f3720e9/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EntityToMapConverter.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EntityToMapConverter.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EntityToMapConverter.java index 98a8ed5..8eb9f48 100644 --- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EntityToMapConverter.java +++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EntityToMapConverter.java @@ -65,6 +65,7 @@ public class EntityToMapConverter { * @param applicationScope * @param entity The entity * @param indexEdge The edge this entity is indexed on + * @param fieldsToIndex A set of fields that will be indexed should they exist on the entity. Other fields will be filtered out. */ public static Map<String, Object> convert(ApplicationScope applicationScope, final IndexEdge indexEdge, final Entity entity, Optional<Set<String>> fieldsToIndex) { @@ -114,7 +115,6 @@ public class EntityToMapConverter { if(fieldsToIndex.isPresent()){ Set<String> defaultProperties = fieldsToIndex.get(); - //copy paste logic here. HashSet mapFields = ( HashSet ) outputEntity.get( "fields" ); Iterator collectionIterator = mapFields.iterator(); @@ -137,10 +137,11 @@ public class EntityToMapConverter { } /** - * This method is crucial for selective top level indexing. Here we check to see if the flatted properties - * are in fact a top level exclusion e.g one.two.three and one.three.two can be allowed for querying by - * specifying one in the schema. If they are not a top level exclusion then they are removed from the iterator and - * the map. + * Handles checking to see if a field is a top level exclusion or just a field that shouldn't be indexed. + * This is handled by looping through all the fields we want to be able to query on, and checking to see if a + * specific field name is included. If the field name is included then do nothing. If the field name is not included + * then we do not want to be able to query on it. Instead we remove it from the collectionIterator which + * removes it from the outputEntity above, thus filtering it out. * * @param fieldsToKeep - contains a list of fields that the user defined in their schema. * @param collectionIterator - contains the iterator with the reference to the map where we want to remove the field. Once removed here it is removed from the entity so it won't be indexed. @@ -154,12 +155,12 @@ public class EntityToMapConverter { - //goes through a loop of all the fields ( excluding default ) that we want to keep. - //if the toRemoveFlag is set to false then we want to keep the property, otherwise we set it to true and remove + //goes through a loop of all the fields that we want to keep. + //if the toRemoveFlag is set to false then we want to keep the property and do nothing to it, otherwise we set it to true and remove //the property. while ( fieldIterator.hasNext() ) { //this is the field that we're - String requiredInclusionString = ( String ) fieldIterator.next(); + String fieldToKeep = ( String ) fieldIterator.next(); //Since we know that the fieldName cannot be equal to the requiredInclusion criteria due to the if condition before we enter this method @@ -169,9 +170,9 @@ public class EntityToMapConverter { //The second part of the if loop also requires that the fieldName is followed by a period after we check to ensure that the //indexing criteria is included in the string. This is done to weed out values such as one.twoexample.three // when we should only keep one.two.three when comparing the indexing criteria of one.two. - if(fieldName.length() > requiredInclusionString.length() - && fieldName.contains( requiredInclusionString ) - && fieldName.charAt( requiredInclusionString.length() )=='.' ) { + if(fieldName.length() > fieldToKeep.length() + && fieldName.contains( fieldToKeep ) + && fieldName.charAt( fieldToKeep.length() )=='.' ) { toRemoveFlag = false; break; }
