Cleaning up the left over code from the endeavor.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/0feeebbc Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/0feeebbc Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/0feeebbc Branch: refs/heads/release-2.1.1 Commit: 0feeebbccd47ebc642925da27164bb00e26a7e54 Parents: 0e53dcf Author: George Reyes <[email protected]> Authored: Tue Mar 29 16:02:29 2016 -0700 Committer: George Reyes <[email protected]> Committed: Tue Mar 29 16:02:29 2016 -0700 ---------------------------------------------------------------------- .../corepersistence/CpRelationManager.java | 9 -- .../asyncevents/EventBuilderImpl.java | 1 - .../corepersistence/index/IndexServiceImpl.java | 115 ++----------------- .../usergrid/persistence/map/MapManager.java | 2 - .../index/impl/EntityToMapConverter.java | 10 -- .../persistence/index/impl/EntityIndexTest.java | 4 +- .../rest/applications/ServiceResource.java | 3 - .../collection/CollectionsResourceIT.java | 20 ---- .../usergrid/services/AbstractService.java | 2 +- .../apache/usergrid/services/ServiceAction.java | 2 +- .../usergrid/services/ServiceManager.java | 2 +- 11 files changed, 13 insertions(+), 157 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/usergrid/blob/0feeebbc/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java ---------------------------------------------------------------------- diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java index 739aea4..378767e 100644 --- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java +++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java @@ -356,15 +356,6 @@ public class CpRelationManager implements RelationManager { @Override public Entity addToCollection( String collectionName, EntityRef itemRef ) throws Exception { - /** - * Get the map manager for uuid mapping - */ - Id mapOwner = new SimpleId( applicationId, TYPE_APPLICATION ); - - final MapScope ms = CpNamingUtils.getEntityTypeMapScope(mapOwner ); - - MapManager mm = managerCache.getMapManager( ms ); - Preconditions.checkNotNull( itemRef, "itemref is null" ); CollectionInfo collection = getDefaultSchema().getCollection( headEntity.getType(), collectionName ); if ( ( collection != null && collection.getType() != null ) && !collection.getType() http://git-wip-us.apache.org/repos/asf/usergrid/blob/0feeebbc/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java ---------------------------------------------------------------------- diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java index f2a8ff3..b6318fd 100644 --- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java +++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java @@ -141,7 +141,6 @@ public class EventBuilderImpl implements EventBuilder { return new EntityDeleteResults( deIndexObservable, ecmDeleteObservable, graphCompactObservable ); } - //TODO: have the map manager here and filter out the entities that are going to be loaded. @Override public Observable<IndexOperationMessage> buildEntityIndex( final EntityIndexOperation entityIndexOperation ) { http://git-wip-us.apache.org/repos/asf/usergrid/blob/0feeebbc/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java ---------------------------------------------------------------------- diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java index c3d5361..c010c06 100644 --- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java +++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java @@ -123,9 +123,8 @@ public class IndexServiceImpl implements IndexService { //do our observable for batching //try to send a whole batch if we can - //TODO: extract the below and call a single method. final Observable<IndexOperationMessage> batches = sourceEdgesToIndex - .buffer(250, TimeUnit.MILLISECONDS, indexFig.getIndexBatchSize() ) //TODO: change to delay. maybe. at least to the before buffer. + .buffer(250, TimeUnit.MILLISECONDS, indexFig.getIndexBatchSize() ) //map into batches based on our buffer size .flatMap( buffer -> Observable.from( buffer ) @@ -135,7 +134,7 @@ public class IndexServiceImpl implements IndexService { logger.debug("adding edge {} to batch for entity {}", indexEdge, entity); } - final Optional<Set<String>> fieldsToIndex = getFilteredStringObjectMap( applicationScope, entity, indexEdge ); + final Optional<Set<String>> fieldsToIndex = getFilteredStringObjectMap( indexEdge ); batch.index( indexEdge, entity ,fieldsToIndex); } ) @@ -168,7 +167,7 @@ public class IndexServiceImpl implements IndexService { logger.debug("adding edge {} to batch for entity {}", indexEdge, entity); } - Optional<Set<String>> fieldsToIndex = getFilteredStringObjectMap( applicationScope, entity, indexEdge ); + Optional<Set<String>> fieldsToIndex = getFilteredStringObjectMap( indexEdge ); batch.index( indexEdge, entity ,fieldsToIndex); @@ -180,30 +179,21 @@ public class IndexServiceImpl implements IndexService { } - /** * This method takes in an entity and flattens it out then begins the process to filter out * properties that we do not want to index. Once flatted we parse through each property * and verify that we want it to be indexed on. The set of default properties that will always be indexed are as follows. * UUID - TYPE - MODIFIED - CREATED. Depending on the schema this may change. For instance, users will always require * NAME, but the above four will always be taken in. - * @param applicationScope - * @param entity + * @param indexEdge * @return This returns a filtered map that contains the flatted properties of the entity. If there isn't a schema * associated with the collection then return null ( and index the entity in its entirety ) */ - private Optional<Set<String>> getFilteredStringObjectMap( final ApplicationScope applicationScope, - final Entity entity, final IndexEdge indexEdge ) { - - - //TODO: THIS IS THE FIRST THING TO BE LOOKED AT TOMORROW. - //look into this. - IndexOperation indexOperation = new IndexOperation(); + private Optional<Set<String>> getFilteredStringObjectMap( final IndexEdge indexEdge ) { Id mapOwner = new SimpleId( indexEdge.getNodeId().getUuid(), TYPE_APPLICATION ); - //TODO: this needs to be cached final MapScope ms = CpNamingUtils.getEntityTypeMapScope( mapOwner ); MapManager mm = mapManagerFactory.createMapManager( ms ); @@ -211,17 +201,16 @@ public class IndexServiceImpl implements IndexService { Set<String> defaultProperties; ArrayList fieldsToKeep; - //TODO: extract collection name using other classes than the split. - String jsonSchemaMap = mm.getString( indexEdge.getEdgeName().split( "\\|" )[1] ); + String collectionName = CpNamingUtils.getCollectionNameFromEdgeName( indexEdge.getEdgeName() ); + String jsonSchemaMap = mm.getString( collectionName ); //If we do have a schema then parse it and add it to a list of properties we want to keep.Otherwise return. if ( jsonSchemaMap != null ) { Map jsonMapData = ( Map ) JsonUtils.parse( jsonSchemaMap ); Schema schema = Schema.getDefaultSchema(); - defaultProperties = schema.getRequiredProperties( indexEdge.getEdgeName().split( "\\|" )[1] ); + defaultProperties = schema.getRequiredProperties( collectionName ); fieldsToKeep = ( ArrayList ) jsonMapData.get( "fields" ); - //TODO: add method here to update the relevant fields in the map manager when it was accessed. defaultProperties.addAll( fieldsToKeep ); } @@ -232,94 +221,6 @@ public class IndexServiceImpl implements IndexService { return Optional.of(defaultProperties); } - - /** - * This method is crucial for selective top level indexing. Here we check to see if the flatted properties - * are in fact a top level exclusion e.g one.two.three and one.three.two can be allowed for querying by - * specifying one in the schema. If they are not a top level exclusion then they are removed from the iterator and - * the map. - * - * @param fieldsToKeep - contains a list of fields that the user defined in their schema. - * @param collectionIterator - contains the iterator with the reference to the map where we want to remove the field. - * @param fieldName - contains the name of the field that we want to keep. - */ - private void iterateThroughMapForFieldsToBeIndexed( final ArrayList fieldsToKeep, final Iterator collectionIterator, - final String fieldName ) { - boolean toRemoveFlag = true; - String[] flattedStringArray = getStrings( fieldName ); - - Iterator fieldIterator = fieldsToKeep.iterator(); - - //goes through a loop of all the fields ( excluding default ) that we want to keep. - //if the toRemoveFlag is set to false then we want to keep the property, otherwise we set it to true and remove - //the property. - while ( fieldIterator.hasNext() ) { - String requiredInclusionString = ( String ) fieldIterator.next(); - String[] flattedRequirementString = getStrings( requiredInclusionString ); - - - //loop each split array value to see if it matches the equivalent value - //in the field. e.g in the example one.two.three and one.two.four we need to check that the schema - //matches in both one and two above. If instead it says to exclude one.twor then we would still exclude the above - //since it is required to be a hard match. - - //The way the below works if we see that the current field isn't as fine grained as the schema rule - //( aka the array is shorter than the current index of the schema rule then there is no way the rule could apply - // to the index. - - //Then if that check passes we go to check that both parts are equal. If they are ever not equal - // e.g one.two.three and one.three.two then it shouldn't be included - //TODO: regex. - for ( int index = 0; index < flattedRequirementString.length; index++ ) { - //if the array contains a string that it is equals to then set the remove flag to true - //otherwise remain false. - - //one.three - //one.two - //one - if ( flattedStringArray.length <= index ) { - toRemoveFlag = true; - break; - } - - if ( flattedRequirementString[index].equals( flattedStringArray[index] ) ) { - toRemoveFlag = false; - } - else { - toRemoveFlag = true; - break; - } - } - if ( toRemoveFlag == false ) { - break; - } - } - - if ( toRemoveFlag ) { - collectionIterator.remove(); - } - } - - - /** - * Splits the string on the flattened period "." seperated values. - * @param fieldName - * @return - */ - private String[] getStrings( final String fieldName ) { - final String[] flattedStringArray; - if ( !fieldName.contains( "." ) ) { - //create a single array that is made of a the single value. - flattedStringArray = new String[] { fieldName }; - } - else { - flattedStringArray = fieldName.split( "\\." ); - } - return flattedStringArray; - } - - - //Steps to delete an IndexEdge. //1.Take the search edge given and search for all the edges in elasticsearch matching that search edge //2. Batch Delete all of those edges returned in the previous search. http://git-wip-us.apache.org/repos/asf/usergrid/blob/0feeebbc/stack/corepersistence/map/src/main/java/org/apache/usergrid/persistence/map/MapManager.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/map/src/main/java/org/apache/usergrid/persistence/map/MapManager.java b/stack/corepersistence/map/src/main/java/org/apache/usergrid/persistence/map/MapManager.java index ca8fd9a..80e2d17 100644 --- a/stack/corepersistence/map/src/main/java/org/apache/usergrid/persistence/map/MapManager.java +++ b/stack/corepersistence/map/src/main/java/org/apache/usergrid/persistence/map/MapManager.java @@ -27,8 +27,6 @@ import java.util.UUID; /** * Generator of a map manager instance */ -//TODO: This should be a singleton, otherwise cache could be out of sync and would need to invalidated everywher - //TODO: make manager cache injectable everywhere. public interface MapManager { http://git-wip-us.apache.org/repos/asf/usergrid/blob/0feeebbc/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EntityToMapConverter.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EntityToMapConverter.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EntityToMapConverter.java index 5a728a3..885f66e 100644 --- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EntityToMapConverter.java +++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EntityToMapConverter.java @@ -108,10 +108,8 @@ public class EntityToMapConverter { //now visit our entity final FieldParser parser = new EntityMappingParser(); - //TODO: to the parsing in the parse method below, that way we don't iterate twice and we filter things out as we see them. final Set<EntityField> fieldsToBeFiltered = parser.parse( entityMap ); - //add our fields outputEntity.put( ENTITY_FIELDS, fieldsToBeFiltered ); @@ -126,14 +124,6 @@ public class EntityToMapConverter { EntityField testedField = ( EntityField ) collectionIterator.next(); String fieldName = ( String ) ( testedField ).get( "name" ); - //Checks to see if the fieldname is a default property. If it is then keep it, otherwise send it to - //be verified the aptly named method - - //one.two.three - //one.two.four - //one.two3.five - //one.two - //fields { one.two } if ( !defaultProperties.contains( fieldName ) ) { iterateThroughMapForFieldsToBeIndexed( defaultProperties, collectionIterator, fieldName ); } http://git-wip-us.apache.org/repos/asf/usergrid/blob/0feeebbc/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java index 27f3afd..008ec80 100644 --- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java +++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java @@ -365,7 +365,7 @@ public class EntityIndexTest extends BaseIT { EntityUtils.setVersion(entity, UUIDGenerator.newTimeUUID() ); entity.setField(new UUIDField(IndexingUtils.ENTITY_ID_FIELDNAME, UUID.randomUUID() ) ); - indexProducer.put(entityIndex.createBatch().index( searchEdge, entity, null ).build()).subscribe(); + indexProducer.put(entityIndex.createBatch().index( searchEdge, entity ).build()).subscribe(); entityIndex.refreshAsync().toBlocking().first(); CandidateResults candidateResults = entityIndex @@ -412,7 +412,7 @@ public class EntityIndexTest extends BaseIT { IndexEdge searchEdge = new IndexEdgeImpl( appId, "mehCars", SearchEdge.NodeType.SOURCE, System.currentTimeMillis()*1000 ); //index the new entity. This is where the loop will be set to create like 100 entities. - indexProducer.put(entityIndex.createBatch().index( searchEdge, entity[i], null ).build()).subscribe(); + indexProducer.put(entityIndex.createBatch().index( searchEdge, entity[i] ).build()).subscribe(); } entityIndex.refreshAsync().toBlocking().first(); http://git-wip-us.apache.org/repos/asf/usergrid/blob/0feeebbc/stack/rest/src/main/java/org/apache/usergrid/rest/applications/ServiceResource.java ---------------------------------------------------------------------- diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/applications/ServiceResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/applications/ServiceResource.java index 9e7bc91..0b2fef6 100644 --- a/stack/rest/src/main/java/org/apache/usergrid/rest/applications/ServiceResource.java +++ b/stack/rest/src/main/java/org/apache/usergrid/rest/applications/ServiceResource.java @@ -462,9 +462,6 @@ public class ServiceResource extends AbstractContextResource { return response; } - - - @SuppressWarnings({ "unchecked" }) public ServicePayload getPayload( Object json ) { ServicePayload payload = null; http://git-wip-us.apache.org/repos/asf/usergrid/blob/0feeebbc/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/CollectionsResourceIT.java ---------------------------------------------------------------------- diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/CollectionsResourceIT.java b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/CollectionsResourceIT.java index 7926e49..04ed18e 100644 --- a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/CollectionsResourceIT.java +++ b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/CollectionsResourceIT.java @@ -152,25 +152,9 @@ public class CollectionsResourceIT extends AbstractRestIT { Entity thing = this.app().collection( "testCollection" ).collection( "_indexes" ).post( payload ); refreshIndex(); - //TODO: write a test to verify the data below. - - // Collection collection = this.app().collection( "testCollection" ).collection( "_index" ).get(); - // - // LinkedHashMap testCollectionSchema = (LinkedHashMap)collection.getResponse().getData(); - // //TODO: the below will have to be replaced by the values that I deem correct. - // assertEquals( ( thing ).get( "lastUpdated" ), testCollectionSchema.get( "lastUpdated" )); - // assertEquals( ( thing ).get( "lastUpdateBy" ),testCollectionSchema.get( "lastUpdateBy" ) ); - // assertEquals( ( thing ).get( "lastReindexed" ),testCollectionSchema.get( "lastReindexed" ) ); - // - // //TODO: this test doesn't check to see if create checks the schema. Only that the reindex removes whats already there. - // ArrayList<String> schema = ( ArrayList<String> ) testCollectionSchema.get( "fields" ); - // assertEquals( "one",schema.get( 0 ) ); - //Reindex and verify that the entity only has field one index. this.app().collection( "testCollection" ).collection( "_reindex" ).post(true,clientSetup.getSuperuserToken(),ApiResponse.class,null,null,false); - // Thread.sleep( 10000 ); - //refreshIndex(); for(int i = 0; i < 10; i++) { String query = "one ='value"+ i + "'"; @@ -218,17 +202,13 @@ public class CollectionsResourceIT extends AbstractRestIT { Entity thing = this.app().collection( "testCollection" ).collection( "_indexes" ).post( payload ); refreshIndex(); - //TODO: write a test to verify the data below. - Collection collection = this.app().collection( "testCollection" ).collection( "_index" ).get(); LinkedHashMap testCollectionSchema = (LinkedHashMap)collection.getResponse().getData(); - //TODO: the below will have to be replaced by the values that I deem correct. assertEquals( ( thing ).get( "lastUpdated" ), testCollectionSchema.get( "lastUpdated" )); assertEquals( ( thing ).get( "lastUpdateBy" ),testCollectionSchema.get( "lastUpdateBy" ) ); assertEquals( 0,testCollectionSchema.get( "lastReindexed" ) ); - //TODO: this test doesn't check to see if create checks the schema. Only that the reindex removes whats already there. ArrayList<String> schema = ( ArrayList<String> ) testCollectionSchema.get( "fields" ); assertEquals( "one",schema.get( 0 ) ); http://git-wip-us.apache.org/repos/asf/usergrid/blob/0feeebbc/stack/services/src/main/java/org/apache/usergrid/services/AbstractService.java ---------------------------------------------------------------------- diff --git a/stack/services/src/main/java/org/apache/usergrid/services/AbstractService.java b/stack/services/src/main/java/org/apache/usergrid/services/AbstractService.java index 7d33144..e10cb3f 100644 --- a/stack/services/src/main/java/org/apache/usergrid/services/AbstractService.java +++ b/stack/services/src/main/java/org/apache/usergrid/services/AbstractService.java @@ -1366,7 +1366,7 @@ public abstract class AbstractService implements Service { if ( currentUser == null ) { return; } - //this is breaking because there is no entity manager there. + String perm = getPermissionFromPath( em.getApplicationRef().getUuid(), context.getAction().toString().toLowerCase(), path ); boolean permitted = currentUser.isPermitted( perm ); http://git-wip-us.apache.org/repos/asf/usergrid/blob/0feeebbc/stack/services/src/main/java/org/apache/usergrid/services/ServiceAction.java ---------------------------------------------------------------------- diff --git a/stack/services/src/main/java/org/apache/usergrid/services/ServiceAction.java b/stack/services/src/main/java/org/apache/usergrid/services/ServiceAction.java index ec34df4..e1c6133 100644 --- a/stack/services/src/main/java/org/apache/usergrid/services/ServiceAction.java +++ b/stack/services/src/main/java/org/apache/usergrid/services/ServiceAction.java @@ -18,5 +18,5 @@ package org.apache.usergrid.services; public enum ServiceAction { - POST, GET, PUT, DELETE, HEAD, OPTIONS, SCHEMA; + POST, GET, PUT, DELETE, HEAD, OPTIONS; } http://git-wip-us.apache.org/repos/asf/usergrid/blob/0feeebbc/stack/services/src/main/java/org/apache/usergrid/services/ServiceManager.java ---------------------------------------------------------------------- diff --git a/stack/services/src/main/java/org/apache/usergrid/services/ServiceManager.java b/stack/services/src/main/java/org/apache/usergrid/services/ServiceManager.java index 9089ab7..a9892f5 100644 --- a/stack/services/src/main/java/org/apache/usergrid/services/ServiceManager.java +++ b/stack/services/src/main/java/org/apache/usergrid/services/ServiceManager.java @@ -290,7 +290,7 @@ public class ServiceManager { if ( ( cls != null ) && !Modifier.isAbstract( cls.getModifiers() ) ) { return cls; } - } //TODO:Find a different way to express this exception. The real exception is returned above. This just needs to be handled some way. + } catch ( ClassNotFoundException e1 ) { if(logger.isTraceEnabled()){ logger.trace("Could not find class", e1);
