move initialize index
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/f69bcebe Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/f69bcebe Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/f69bcebe Branch: refs/heads/USERGRID-347 Commit: f69bcebe4439a408a4d2baa87a3322142c506f93 Parents: 2af7489 Author: Shawn Feldman <[email protected]> Authored: Fri Mar 20 17:22:22 2015 -0600 Committer: Shawn Feldman <[email protected]> Committed: Fri Mar 20 17:22:22 2015 -0600 ---------------------------------------------------------------------- .../corepersistence/CpEntityManagerFactory.java | 13 +++++++--- .../usergrid/corepersistence/CpSetup.java | 13 +++++++++- .../index/ApplicationEntityIndex.java | 5 ++++ .../usergrid/persistence/index/EntityIndex.java | 6 +---- .../impl/EsApplicationEntityIndexImpl.java | 12 +++++++++ .../index/impl/EsEntityIndexImpl.java | 9 ------- .../persistence/index/impl/EntityIndexTest.java | 26 +++++++++++--------- .../index/impl/IndexLoadTestsIT.java | 4 +-- 8 files changed, 56 insertions(+), 32 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f69bcebe/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java ---------------------------------------------------------------------- diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java index 1388462..95ffbaa 100644 --- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java +++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java @@ -25,7 +25,9 @@ import java.util.TreeMap; import java.util.UUID; import java.util.concurrent.atomic.AtomicBoolean; +import com.hazelcast.core.IdGenerator; import org.apache.usergrid.persistence.index.ApplicationEntityIndex; +import org.apache.usergrid.persistence.index.EntityIndexFactory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.BeansException; @@ -92,6 +94,7 @@ import static org.apache.usergrid.persistence.Schema.TYPE_APPLICATION; public class CpEntityManagerFactory implements EntityManagerFactory, ApplicationContextAware { private static final Logger logger = LoggerFactory.getLogger( CpEntityManagerFactory.class ); + private final EntityIndexFactory entityIndexFactory; private ApplicationContext applicationContext; @@ -129,6 +132,7 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application this.counterUtils = counterUtils; this.injector = injector; this.entityIndex = injector.getInstance(EntityIndex.class); + this.entityIndexFactory = injector.getInstance(EntityIndexFactory.class); this.managerCache = injector.getInstance( ManagerCache.class ); this.metricsFactory = injector.getInstance( MetricsFactory.class ); @@ -160,7 +164,9 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application em.getApplication(); } - entityIndex.initializeIndex(); + ApplicationScope appScope = new ApplicationScopeImpl(new SimpleId( CpNamingUtils.SYSTEM_APP_ID, "application" ) ); + ApplicationEntityIndex applicationEntityIndex = entityIndexFactory.createApplicationEntityIndex(appScope); + applicationEntityIndex.initializeIndex(); entityIndex.refresh(); @@ -722,9 +728,10 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application public void rebuildApplicationIndexes( UUID appId, ProgressObserver po ) throws Exception { EntityManager em = getEntityManager( appId ); - + ApplicationScope applicationScope = new ApplicationScopeImpl( new SimpleId( CpNamingUtils.SYSTEM_APP_ID, "application" )); //explicitly invoke create index, we don't know if it exists or not in ES during a rebuild. - entityIndex.initializeIndex(); + ApplicationEntityIndex applicationEntityIndex = entityIndexFactory.createApplicationEntityIndex(applicationScope); + applicationEntityIndex.initializeIndex(); em.reindex(po); em.reindex( po ); http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f69bcebe/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpSetup.java ---------------------------------------------------------------------- diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpSetup.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpSetup.java index a9e6cee..1d75e47 100644 --- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpSetup.java +++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpSetup.java @@ -20,7 +20,13 @@ package org.apache.usergrid.corepersistence; import java.util.UUID; import com.google.inject.Binding; +import org.apache.usergrid.corepersistence.util.CpNamingUtils; +import org.apache.usergrid.persistence.core.scope.ApplicationScope; +import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl; +import org.apache.usergrid.persistence.index.ApplicationEntityIndex; import org.apache.usergrid.persistence.index.EntityIndex; +import org.apache.usergrid.persistence.index.EntityIndexFactory; +import org.apache.usergrid.persistence.model.entity.SimpleId; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -67,6 +73,7 @@ public class CpSetup implements Setup { private final EntityManagerFactory emf; private final EntityIndex entityIndex; + private final EntityIndexFactory entityIndexFactory; /** @@ -79,6 +86,8 @@ public class CpSetup implements Setup { this.cass = cassandraService; this.injector = injector; this.entityIndex = injector.getInstance(EntityIndex.class); + this.entityIndexFactory = injector.getInstance(EntityIndexFactory.class); + } @@ -98,8 +107,10 @@ public class CpSetup implements Setup { setupStaticKeyspace(); + ApplicationScope applicationScope = new ApplicationScopeImpl(new SimpleId( CpNamingUtils.SYSTEM_APP_ID,"application")); + ApplicationEntityIndex applicationEntityIndex = entityIndexFactory.createApplicationEntityIndex(applicationScope); //force the EMF creation of indexes before creating the default applications - entityIndex.initializeIndex(); + applicationEntityIndex.initializeIndex(); injector.getInstance( DataMigrationManager.class ).migrate(); http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f69bcebe/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/ApplicationEntityIndex.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/ApplicationEntityIndex.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/ApplicationEntityIndex.java index 34967bd..4808d8a 100644 --- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/ApplicationEntityIndex.java +++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/ApplicationEntityIndex.java @@ -32,6 +32,11 @@ public interface ApplicationEntityIndex { /** + * + */ + public void initializeIndex(); + + /** * Create the index batch. */ public EntityIndexBatch createBatch(); http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f69bcebe/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java index 854c976..af6b013 100644 --- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java +++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java @@ -38,11 +38,7 @@ import java.util.concurrent.Future; */ public interface EntityIndex extends CPManager { - /** - * This should ONLY ever be called once on application create. - * Otherwise we're introducing slowness into our system - */ - public void initializeIndex(); + /** * Create an index and add to alias, will create alias and remove any old index from write alias if alias already exists * @param indexSuffix index name http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f69bcebe/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java index 56f8030..2d01697 100644 --- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java +++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java @@ -126,6 +126,18 @@ public class EsApplicationEntityIndexImpl implements ApplicationEntityIndex{ } @Override + public void initializeIndex() { + final int numberOfShards = indexFig.getNumberOfShards(); + final int numberOfReplicas = indexFig.getNumberOfReplicas(); + indexCache.invalidate(alias); + String[] indexes = entityIndex.getUniqueIndexes(); + if(indexes == null || indexes.length==0) { + entityIndex.addIndex(null, numberOfShards, numberOfReplicas, indexFig.getWriteConsistencyLevel()); + } + + } + + @Override public EntityIndexBatch createBatch() { EntityIndexBatch batch = new EsEntityIndexBatchImpl( applicationScope, indexBatchBufferProducer, entityIndex, indexIdentifier ); http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f69bcebe/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java index 4290f23..259fa55 100644 --- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java +++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java @@ -139,16 +139,7 @@ public class EsEntityIndexImpl implements AliasedEntityIndex { } - @Override - public void initializeIndex() { - final int numberOfShards = config.getNumberOfShards(); - final int numberOfReplicas = config.getNumberOfReplicas(); - String[] indexes = getIndexesFromEs(AliasType.Write); - if(indexes == null || indexes.length==0) { - addIndex(null, numberOfShards, numberOfReplicas, config.getWriteConsistencyLevel()); - } - } @Override public void addIndex(final String indexSuffix,final int numberOfShards, final int numberOfReplicas, final String writeConsistency) { http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f69bcebe/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java index b562ccf..85570f1 100644 --- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java +++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java @@ -91,10 +91,10 @@ public class EntityIndexTest extends BaseIT { @Test public void testIndex() throws IOException, InterruptedException { Id appId = new SimpleId("application"); - ei.initializeIndex(); ApplicationScope applicationScope = new ApplicationScopeImpl(appId); ApplicationEntityIndex entityIndex = eif.createApplicationEntityIndex(applicationScope); + entityIndex.initializeIndex(); final String entityType = "thing"; IndexScope indexScope = new IndexScopeImpl(appId, "things"); @@ -158,7 +158,7 @@ public class EntityIndexTest extends BaseIT { final ApplicationEntityIndex entityIndex = eif.createApplicationEntityIndex(applicationScope); final IndexScope indexScope = new IndexScopeImpl(appId, "things"); final String entityType = "thing"; - ei.initializeIndex(); + entityIndex.initializeIndex(); final CountDownLatch latch = new CountDownLatch(threads); final AtomicLong failTime=new AtomicLong(0); InputStream is = this.getClass().getResourceAsStream( "/sample-large.json" ); @@ -201,9 +201,9 @@ public class EntityIndexTest extends BaseIT { ApplicationScope applicationScope = new ApplicationScopeImpl( appId ); ApplicationEntityIndex entityIndex = eif.createApplicationEntityIndex(applicationScope); - ei.initializeIndex(); + entityIndex.initializeIndex(); for(int i=0;i<10;i++) { - ei.initializeIndex(); + entityIndex.initializeIndex(); } } @@ -215,7 +215,7 @@ public class EntityIndexTest extends BaseIT { ApplicationScope applicationScope = new ApplicationScopeImpl( appId ); ApplicationEntityIndex entityIndex = eif.createApplicationEntityIndex(applicationScope); - ei.initializeIndex(); + entityIndex.initializeIndex(); final String entityType = "thing"; IndexScope indexScope = new IndexScopeImpl( appId, "things" ); @@ -328,7 +328,7 @@ public class EntityIndexTest extends BaseIT { IndexScope indexScope = new IndexScopeImpl( appId, "fastcars" ); ApplicationEntityIndex entityIndex = eif.createApplicationEntityIndex(applicationScope); - ei.initializeIndex(); + entityIndex.initializeIndex(); Map entityMap = new HashMap() {{ put( "name", "Ferrari 212 Inter" ); @@ -459,7 +459,7 @@ public class EntityIndexTest extends BaseIT { IndexScope appScope = new IndexScopeImpl( ownerId, "user" ); ApplicationEntityIndex entityIndex = eif.createApplicationEntityIndex(applicationScope); - ei.initializeIndex(); + entityIndex.initializeIndex(); final String middleName = "middleName" + UUIDUtils.newTimeUUID(); @@ -508,7 +508,7 @@ public class EntityIndexTest extends BaseIT { IndexScope appScope = new IndexScopeImpl( ownerId, "user" ); ApplicationEntityIndex entityIndex = eif.createApplicationEntityIndex(applicationScope); - ei.initializeIndex(); + entityIndex.initializeIndex(); entityIndex.createBatch(); // Bill has favorites as string, age as string and retirement goal as number @@ -576,11 +576,13 @@ public class EntityIndexTest extends BaseIT { @Test public void healthTest() { - + Id appId = new SimpleId( "entityindextest" ); + Id ownerId = new SimpleId( "multivaluedtype" ); + ApplicationScope applicationScope = new ApplicationScopeImpl( appId ); assertNotEquals( "cluster should be ok", Health.RED, ei.getClusterHealth() ); assertEquals( "index should be ready", Health.GREEN, ei.getIndexHealth() ); - - ei.initializeIndex(); + ApplicationEntityIndex entityIndex = eif.createApplicationEntityIndex(applicationScope); + entityIndex.initializeIndex(); ei.refresh(); assertNotEquals( "cluster should be fine", Health.RED, ei.getIndexHealth() ); @@ -600,7 +602,7 @@ public class EntityIndexTest extends BaseIT { ApplicationEntityIndex entityIndex = eif.createApplicationEntityIndex(applicationScope); - ei.initializeIndex(); + entityIndex.initializeIndex(); final EntityIndexBatch batch = entityIndex.createBatch(); http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f69bcebe/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/IndexLoadTestsIT.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/IndexLoadTestsIT.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/IndexLoadTestsIT.java index 623f9dd..44a9743 100644 --- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/IndexLoadTestsIT.java +++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/IndexLoadTestsIT.java @@ -81,12 +81,12 @@ public class IndexLoadTestsIT extends BaseIT { final Id applicationId = new SimpleId( applicationUUID, "application" ); final ApplicationScope scope = new ApplicationScopeImpl( applicationId ); + ApplicationEntityIndex applicationEntityIndex = entityIndexFactory.createApplicationEntityIndex(scope); //create our index if it doesn't exist - index.initializeIndex(); + applicationEntityIndex.initializeIndex(); - ApplicationEntityIndex applicationEntityIndex = entityIndexFactory.createApplicationEntityIndex(scope); final Observable<Entity> createEntities = createStreamFromWorkers( applicationEntityIndex, applicationId ); //run them all
