Repository: incubator-usergrid Updated Branches: refs/heads/USERGRID-347 d7734af90 -> d263d36d8
single index Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/346eba98 Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/346eba98 Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/346eba98 Branch: refs/heads/USERGRID-347 Commit: 346eba98ed1080940730fb617951e6172a1683bb Parents: 1199e47 Author: Shawn Feldman <[email protected]> Authored: Wed Mar 18 14:56:36 2015 -0600 Committer: Shawn Feldman <[email protected]> Committed: Wed Mar 18 14:56:36 2015 -0600 ---------------------------------------------------------------------- .../corepersistence/CpEntityManagerFactory.java | 3 - .../usergrid/persistence/index/EntityIndex.java | 21 ------ .../persistence/index/IndexIdentifier.java | 16 +---- .../usergrid/persistence/index/SearchTypes.java | 14 +++- .../index/impl/EsEntityIndexBatchImpl.java | 6 +- .../index/impl/EsEntityIndexImpl.java | 69 +++++--------------- .../persistence/index/impl/IndexingUtils.java | 8 ++- .../persistence/index/impl/EntityIndexTest.java | 65 +----------------- 8 files changed, 42 insertions(+), 160 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/346eba98/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java ---------------------------------------------------------------------- diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java index fe4d828..2bdb54c 100644 --- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java +++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java @@ -292,7 +292,6 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application EntityManager appEm = getEntityManager( applicationId ); appEm.create( applicationId, TYPE_APPLICATION, properties ); - appEm.createIndex(); appEm.resetRoles(); appEm.refreshIndex(); @@ -706,7 +705,6 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application EntityManager em = getEntityManager( appId ); //explicitly invoke create index, we don't know if it exists or not in ES during a rebuild. - em.createIndex(); Application app = em.getApplication(); em.reindex( po ); @@ -760,7 +758,6 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application EntityManager em = getEntityManager( appId ); //explicitly invoke create index, we don't know if it exists or not in ES during a rebuild. - em.createIndex(); Application app = em.getApplication(); em.reindexCollection( po, collectionName, reverse ); http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/346eba98/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java index aa2bf8e..c8840fa 100644 --- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java +++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java @@ -68,27 +68,6 @@ public interface EntityIndex { public CandidateResults search(final IndexScope indexScope, final SearchTypes searchType, Query query ); /** - * Get the candidate results of all versions of the entity for this id. - * @param indexScope The scope of the index to search in - * @param id The id to search within. - */ - public CandidateResults getEntityVersions(final IndexScope indexScope, final Id id); - -// /** -// * Create a delete method that deletes by Id. This will delete all documents from ES with the same entity Id, -// * effectively removing all versions of an entity from all index scopes -// * @param entityId The entityId to remove -// */ -// public Future deleteAllVersionsOfEntity(final Id entityId ); -// -// /** -// * Takes all the previous versions of the current entity and deletes all previous versions -// * @param id The id to remove -// * @param version The max version to retain -// */ -// public Future deletePreviousVersions(final Id id, final UUID version); - - /** * Refresh the index. */ public void refresh(); http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/346eba98/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexIdentifier.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexIdentifier.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexIdentifier.java index f23dc36..8465d84 100644 --- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexIdentifier.java +++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexIdentifier.java @@ -40,7 +40,7 @@ public class IndexIdentifier{ * @return */ public IndexAlias getAlias() { - return new IndexAlias(config,getIndexBase()); + return new IndexAlias(config,config.getIndexPrefix()); } /** @@ -50,22 +50,12 @@ public class IndexIdentifier{ */ public String getIndex(String suffix) { if (suffix != null) { - return getIndexBase() + "_" + suffix; + return config.getIndexPrefix() + "_" + suffix; } else { - return getIndexBase(); + return config.getIndexPrefix(); } } - /** - * returns the base name for index which will be used to add an alias and index - * @return - */ - private String getIndexBase() { - StringBuilder sb = new StringBuilder(); - sb.append(config.getIndexPrefix()).append(IndexingUtils.SEPARATOR); - IndexingUtils.idString(sb, applicationScope.getApplication()); - return sb.toString(); - } public class IndexAlias{ private final String readAlias; http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/346eba98/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/SearchTypes.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/SearchTypes.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/SearchTypes.java index 35b3a8b..1a97da2 100644 --- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/SearchTypes.java +++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/SearchTypes.java @@ -20,6 +20,10 @@ package org.apache.usergrid.persistence.index;/* */ +import org.apache.usergrid.persistence.core.scope.ApplicationScope; +import org.apache.usergrid.persistence.index.impl.IndexingUtils; +import org.apache.usergrid.persistence.model.entity.Id; + import java.util.Arrays; @@ -37,11 +41,17 @@ public class SearchTypes { private SearchTypes( final String... types ) {this.types = types;} - public String[] getTypeNames() { - return types; + public String[] getTypeNames(ApplicationScope applicationScope) { + String[] typeNames = new String[types.length]; + int i =0 ; + for(String type : types){ + typeNames[i++] = IndexingUtils.getType(applicationScope,type); + } + return typeNames; } + /** * Create a search that will search on the specified types * @param types http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/346eba98/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java index 23b9041..977fe51 100644 --- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java +++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java @@ -117,9 +117,7 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch { String indexId = createIndexDocId( entity, context ); log.debug( "Indexing entity documentId {} data {} ", indexId, entityAsMap ); - final String entityType = entity.getId().getType(); - - + final String entityType =IndexingUtils.getType(applicationScope, entity.getId()); container.addIndexRequest(new IndexRequest(alias.getWriteAlias(), entityType, indexId, entityAsMap)); return this; @@ -134,7 +132,7 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch { ValidationUtils.verifyVersion( version ); final String context = createContextName(indexScope); - final String entityType = id.getType(); + final String entityType =IndexingUtils.getType(applicationScope, id); final String indexId = createIndexDocId( id, version, context ); http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/346eba98/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java index 598c8d9..110b320 100644 --- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java +++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java @@ -199,9 +199,9 @@ public class EsEntityIndexImpl implements AliasedEntityIndex { final int numberOfShards = config.getNumberOfShards(); final int numberOfReplicas = config.getNumberOfReplicas(); String[] indexes = getIndexesFromEs(AliasType.Write); - if(indexes == null || indexes.length==0) { +// if(indexes == null || indexes.length==0) { addIndex(null, numberOfShards, numberOfReplicas, config.getWriteConsistencyLevel()); - } +// } } @Override @@ -233,21 +233,21 @@ public class EsEntityIndexImpl implements AliasedEntityIndex { //ONLY add the alias if we create the index, otherwise we're going to overwrite production settings - /** - * DO NOT MOVE THIS LINE OF CODE UNLESS YOU REALLY KNOW WHAT YOU'RE DOING!!!! - */ - - //We do NOT want to create an alias if the index already exists, we'll overwrite the indexes that - //may have been set via other administrative endpoint - - addAlias(normalizedSuffix); - - testNewIndex(); logger.info("Created new Index Name [{}] ACK=[{}]", indexName, cir.isAcknowledged()); } catch (IndexAlreadyExistsException e) { logger.info("Index Name [{}] already exists", indexName); } + /** + * DO NOT MOVE THIS LINE OF CODE UNLESS YOU REALLY KNOW WHAT YOU'RE DOING!!!! + */ + + //We do NOT want to create an alias if the index already exists, we'll overwrite the indexes that + //may have been set via other administrative endpoint + + addAlias(normalizedSuffix); + + testNewIndex(); @@ -347,7 +347,7 @@ public class EsEntityIndexImpl implements AliasedEntityIndex { // to receive documents. Occasionally we see errors. // See this post: http://s.apache.org/index-missing-exception - logger.debug( "Testing new index name: read {} write {}", alias.getReadAlias(), alias.getWriteAlias()); + logger.debug("Testing new index name: read {} write {}", alias.getReadAlias(), alias.getWriteAlias()); final RetryOperation retryOperation = new RetryOperation() { @Override @@ -410,12 +410,9 @@ public class EsEntityIndexImpl implements AliasedEntityIndex { public CandidateResults search(final IndexScope indexScope, final SearchTypes searchTypes, final Query query ) { - final String context = IndexingUtils.createContextName( indexScope ); - final String[] entityTypes = searchTypes.getTypeNames(); - + final String context = IndexingUtils.createContextName(indexScope); + final String[] entityTypes = searchTypes.getTypeNames(applicationScope); QueryBuilder qb = query.createQueryBuilder(context); - - SearchResponse searchResponse; if ( query.getCursor() == null ) { @@ -635,42 +632,6 @@ public class EsEntityIndexImpl implements AliasedEntityIndex { } - @Override - public CandidateResults getEntityVersions( final IndexScope scope, final Id id ) { - - //since we don't have paging inputs, there's no point in executing a query for paging. - - final String context = IndexingUtils.createContextName(scope); - final SearchTypes searchTypes = SearchTypes.fromTypes(id.getType()); - - final QueryBuilder queryBuilder = - QueryBuilders.termQuery( IndexingUtils.ENTITY_CONTEXT_FIELDNAME, context ); - - final SearchRequestBuilder srb = esProvider.getClient().prepareSearch( alias.getReadAlias() ) - .setTypes(searchTypes.getTypeNames()) - .setScroll(cursorTimeout + "m") - .setQuery(queryBuilder); - - final SearchResponse searchResponse; - try { - //Added For Graphite Metrics - Timer.Context timeEntityIndex = getVersionsTimer.time(); - searchResponse = srb.execute().actionGet(); - timeEntityIndex.stop(); - } - catch ( Throwable t ) { - logger.error( "Unable to communicate with elasticsearch" ); - failureMonitor.fail( "Unable to execute batch", t); - throw t; - } - - - failureMonitor.success(); - - return parseResults(searchResponse, new Query()); - } - - /** * Completely delete an index. http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/346eba98/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java index 0df3a2c..aa6d0f3 100644 --- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java +++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java @@ -230,7 +230,7 @@ public class IndexingUtils { .startObject("template__long") .field("match", LONG_PREFIX + "*") .field("match_mapping_type", "long") - .startObject("mapping").field("type", "long").field("index","not_analyzed").field(DOC_VALUES_KEY, true).endObject() + .startObject("mapping").field("type", "long").field("index", "not_analyzed").field(DOC_VALUES_KEY, true).endObject() .endObject() .endObject() @@ -249,5 +249,11 @@ public class IndexingUtils { } + public static String getType(ApplicationScope applicationScope, Id entityId) { + return getType(applicationScope,entityId.getType()); + } + public static String getType(ApplicationScope applicationScope, String type) { + return idString(applicationScope.getApplication()) + SEPARATOR + type; + } } http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/346eba98/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java index ca9bf79..22ca51f 100644 --- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java +++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java @@ -271,7 +271,7 @@ public class EntityIndexTest extends BaseIT { ObjectMapper mapper = new ObjectMapper(); List<Object> sampleJson = mapper.readValue( is, new TypeReference<List<Object>>() {} ); EntityIndexBatch batch = entityIndex.createBatch(); - insertJsonBlob(sampleJson,batch, entityType, indexScope, max,startIndex); + insertJsonBlob(sampleJson,batch, entityType, indexScope, max, startIndex); batch.execute().get(); entityIndex.refresh(); } @@ -297,10 +297,9 @@ public class EntityIndexTest extends BaseIT { EntityUtils.setVersion(entity, UUIDGenerator.newTimeUUID()); entity.setField(new UUIDField(IndexingUtils.ENTITYID_ID_FIELDNAME, UUID.randomUUID())); batch.index(indexScope, entity); + batch.execute().get(); + - if(count %1000 == 0){ - batch.execute().get(); - } if ( ++count > max ) { break; } @@ -370,7 +369,6 @@ public class EntityIndexTest extends BaseIT { private void testQueries(final IndexScope scope, SearchTypes searchTypes, final EntityIndex entityIndex ) { - testQuery(scope, searchTypes, entityIndex, "name = 'Morgan Pierce'", 1 ); testQuery(scope, searchTypes, entityIndex, "name = 'morgan pierce'", 1 ); @@ -443,63 +441,6 @@ public class EntityIndexTest extends BaseIT { } } - - @Test - public void getEntityVersions() throws Exception { - - Id appId = new SimpleId( "application" ); - Id ownerId = new SimpleId( "owner" ); - - ApplicationScope applicationScope = new ApplicationScopeImpl( appId ); - - IndexScope indexScope = new IndexScopeImpl( ownerId, "users" ); - - - - EntityIndex entityIndex = eif.createEntityIndex( applicationScope ); - entityIndex.initializeIndex(); - - final String middleName = "middleName" + UUIDUtils.newTimeUUID(); - Map<String, Object> properties = new LinkedHashMap<String, Object>(); - properties.put( "username", "edanuff" ); - properties.put( "email", "[email protected]" ); - properties.put( "middlename", middleName ); - - Map entityMap = new HashMap() {{ - put( "username", "edanuff" ); - put( "email", "[email protected]" ); - put( "middlename", middleName ); - }}; - - final Id userId = new SimpleId("user"); - - Entity user = EntityIndexMapUtils.fromMap( entityMap ); - EntityUtils.setId( user, userId); - EntityUtils.setVersion( user, UUIDGenerator.newTimeUUID() ); - - - final EntityIndexBatch batch = entityIndex.createBatch(); - user.setField(new UUIDField(IndexingUtils.ENTITYID_ID_FIELDNAME, UUID.randomUUID())); - - batch.index( indexScope, user ); - - user.setField( new StringField( "address1", "1782 address st" ) ); - batch.index( indexScope, user ); - user.setField( new StringField( "address2", "apt 508" ) ); - batch.index( indexScope, user ); - user.setField( new StringField( "address3", "apt 508" ) ); - batch.index( indexScope, user); - batch.execute().get(); - entityIndex.refresh(); - - CandidateResults results = entityIndex.getEntityVersions(indexScope, user.getId() ); - - assertEquals(1, results.size()); - assertEquals( results.get( 0 ).getId(), user.getId() ); - assertEquals( results.get(0).getVersion(), user.getVersion()); - } - - @Test public void deleteVerification() throws Throwable {
