Repository: incubator-usergrid Updated Branches: refs/heads/USERGRID-501 3a3183662 -> 07cb9dfd6
refactor initiaization index logic Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/07cb9dfd Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/07cb9dfd Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/07cb9dfd Branch: refs/heads/USERGRID-501 Commit: 07cb9dfd6563d974d8c6b9764633be44e389a4d0 Parents: 3a31836 Author: Shawn Feldman <[email protected]> Authored: Mon Mar 30 11:45:15 2015 -0600 Committer: Shawn Feldman <[email protected]> Committed: Mon Mar 30 11:45:15 2015 -0600 ---------------------------------------------------------------------- .../corepersistence/CpEntityManagerFactory.java | 14 +-------- .../index/ApplicationEntityIndex.java | 5 ---- .../usergrid/persistence/index/EntityIndex.java | 10 +++---- .../impl/EsApplicationEntityIndexImpl.java | 12 -------- .../index/impl/EsEntityIndexImpl.java | 13 +++++++++ .../migration/EsIndexDataMigrationImpl.java | 4 ++- .../persistence/index/impl/EntityIndexTest.java | 30 +++++++++++--------- .../index/impl/IndexLoadTestsIT.java | 4 --- 8 files changed, 38 insertions(+), 54 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/07cb9dfd/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java ---------------------------------------------------------------------- diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java index 3cfc1a4..99204a8 100644 --- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java +++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java @@ -164,9 +164,7 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application em.getApplication(); } - ApplicationScope appScope = new ApplicationScopeImpl(new SimpleId( CpNamingUtils.SYSTEM_APP_ID, "application" ) ); - ApplicationEntityIndex applicationEntityIndex = entityIndexFactory.createApplicationEntityIndex(appScope); - applicationEntityIndex.initializeIndex(); + entityIndex.initialize(); entityIndex.refresh(); @@ -265,10 +263,6 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application throw new ApplicationAlreadyExistsException( appName ); } - ApplicationScope applicationScope = new ApplicationScopeImpl(new SimpleId( applicationId,"application")); - ApplicationEntityIndex applicationEntityIndex = entityIndexFactory.createApplicationEntityIndex(applicationScope); - applicationEntityIndex.initializeIndex(); - getSetup().setupApplicationKeyspace( applicationId, appName ); final Optional<UUID> cachedValue = orgApplicationCache.getOrganizationId( organizationName ); @@ -730,12 +724,6 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application public void rebuildApplicationIndexes( UUID appId, ProgressObserver po ) throws Exception { EntityManager em = getEntityManager( appId ); - ApplicationScope applicationScope = new ApplicationScopeImpl( new SimpleId( CpNamingUtils.SYSTEM_APP_ID, "application" )); - //explicitly invoke create index, we don't know if it exists or not in ES during a rebuild. - ApplicationEntityIndex applicationEntityIndex = entityIndexFactory.createApplicationEntityIndex(applicationScope); - applicationEntityIndex.initializeIndex(); - em.reindex(po); - em.reindex( po ); logger.info("\n\nRebuilt index for applicationId {} \n", appId ); http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/07cb9dfd/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/ApplicationEntityIndex.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/ApplicationEntityIndex.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/ApplicationEntityIndex.java index cb1b6f0..86e97c5 100644 --- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/ApplicationEntityIndex.java +++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/ApplicationEntityIndex.java @@ -30,11 +30,6 @@ public interface ApplicationEntityIndex { /** - * - */ - public void initializeIndex(); - - /** * Create the index batch. */ public EntityIndexBatch createBatch(); http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/07cb9dfd/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java index 6783fb0..ff9008e 100644 --- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java +++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java @@ -48,28 +48,28 @@ public interface EntityIndex extends CPManager { * @param replicas * @param writeConsistency */ - public void addIndex(final String indexSuffix, final int shards, final int replicas, final String writeConsistency); + void addIndex(final String indexSuffix, final int shards, final int replicas, final String writeConsistency); /** * Refresh the index. */ - public void refresh(); + void refresh(); /** * Check health of cluster. */ - public Health getClusterHealth(); + Health getClusterHealth(); /** * Check health of this specific index. */ - public Health getIndexHealth(); - + Health getIndexHealth(); + void initialize(); } http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/07cb9dfd/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java index 9248980..84845d0 100644 --- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java +++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java @@ -116,18 +116,6 @@ public class EsApplicationEntityIndexImpl implements ApplicationEntityIndex{ } @Override - public void initializeIndex() { - final int numberOfShards = indexFig.getNumberOfShards(); - final int numberOfReplicas = indexFig.getNumberOfReplicas(); - indexCache.invalidate(alias); - String[] indexes = entityIndex.getUniqueIndexes(); - if(indexes == null || indexes.length==0) { - entityIndex.addIndex(null, numberOfShards, numberOfReplicas, indexFig.getWriteConsistencyLevel()); - } - - } - - @Override public EntityIndexBatch createBatch() { EntityIndexBatch batch = new EsEntityIndexBatchImpl( applicationScope, indexBatchBufferProducer, entityIndex, indexIdentifier ); http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/07cb9dfd/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java index 055af2a..ee28939 100644 --- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java +++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java @@ -131,6 +131,19 @@ public class EsEntityIndexImpl implements AliasedEntityIndex,VersionedData { } @Override + public void initialize(){ + final int numberOfShards = indexFig.getNumberOfShards(); + final int numberOfReplicas = indexFig.getNumberOfReplicas(); + aliasCache.invalidate(alias); + String[] reads = getIndexes(AliasedEntityIndex.AliasType.Read); + String[] writes = getIndexes(AliasedEntityIndex.AliasType.Write); + + if(reads.length==0 || writes.length==0) { + addIndex(null, numberOfShards, numberOfReplicas, indexFig.getWriteConsistencyLevel()); + } + } + + @Override public void addIndex(final String indexSuffix,final int numberOfShards, final int numberOfReplicas, final String writeConsistency) { try { //get index name with suffix attached http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/07cb9dfd/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/EsIndexDataMigrationImpl.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/EsIndexDataMigrationImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/EsIndexDataMigrationImpl.java index 123527a..3706722 100644 --- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/EsIndexDataMigrationImpl.java +++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/EsIndexDataMigrationImpl.java @@ -62,6 +62,8 @@ public class EsIndexDataMigrationImpl implements DataMigration<ApplicationScope> final AdminClient adminClient = provider.getClient().admin(); final int latestVersion = dataVersion.getImplementationVersion(); + entityIndex.initialize(); + observer.start(); try { migrationDataProvider.getData().flatMap(applicationScope -> { @@ -74,7 +76,7 @@ public class EsIndexDataMigrationImpl implements DataMigration<ApplicationScope> aliasesRequestBuilder = adminClient.indices().prepareAliases(); // add read alias try { - aliasesRequestBuilder.addAlias(index, indexIdentifier.getAlias().getReadAlias()); + aliasesRequestBuilder.addAlias(index, indexIdentifier.getAlias().getReadAlias()).get(); } catch (InvalidAliasNameException e) { log.debug("Failed to add alias due to name conflict",e); } http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/07cb9dfd/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java index 2c03635..27f0b9d 100644 --- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java +++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java @@ -30,9 +30,7 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.atomic.AtomicLong; import org.apache.usergrid.persistence.index.*; -import org.junit.Ignore; -import org.junit.Rule; -import org.junit.Test; +import org.junit.*; import org.junit.runner.RunWith; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -87,6 +85,10 @@ public class EntityIndexTest extends BaseIT { @Rule public MigrationManagerRule migrationManagerRule; + @Before + public void setup(){ + ei.initialize(); + } @Test public void testIndex() throws IOException, InterruptedException { @@ -94,7 +96,7 @@ public class EntityIndexTest extends BaseIT { ApplicationScope applicationScope = new ApplicationScopeImpl(appId); ApplicationEntityIndex entityIndex = eif.createApplicationEntityIndex(applicationScope); - entityIndex.initializeIndex(); + final String entityType = "thing"; IndexScope indexScope = new IndexScopeImpl(appId, "things"); @@ -158,7 +160,7 @@ public class EntityIndexTest extends BaseIT { final ApplicationEntityIndex entityIndex = eif.createApplicationEntityIndex(applicationScope); final IndexScope indexScope = new IndexScopeImpl(appId, "things"); final String entityType = "thing"; - entityIndex.initializeIndex(); + final CountDownLatch latch = new CountDownLatch(threads); final AtomicLong failTime=new AtomicLong(0); InputStream is = this.getClass().getResourceAsStream( "/sample-large.json" ); @@ -201,9 +203,9 @@ public class EntityIndexTest extends BaseIT { ApplicationScope applicationScope = new ApplicationScopeImpl( appId ); ApplicationEntityIndex entityIndex = eif.createApplicationEntityIndex(applicationScope); - entityIndex.initializeIndex(); + for(int i=0;i<10;i++) { - entityIndex.initializeIndex(); + } } @@ -215,7 +217,7 @@ public class EntityIndexTest extends BaseIT { ApplicationScope applicationScope = new ApplicationScopeImpl( appId ); ApplicationEntityIndex entityIndex = eif.createApplicationEntityIndex(applicationScope); - entityIndex.initializeIndex(); + final String entityType = "thing"; IndexScope indexScope = new IndexScopeImpl( appId, "things" ); @@ -328,7 +330,7 @@ public class EntityIndexTest extends BaseIT { IndexScope indexScope = new IndexScopeImpl( appId, "fastcars" ); ApplicationEntityIndex entityIndex = eif.createApplicationEntityIndex(applicationScope); - entityIndex.initializeIndex(); + Map entityMap = new HashMap() {{ put( "name", "Ferrari 212 Inter" ); @@ -460,7 +462,7 @@ public class EntityIndexTest extends BaseIT { IndexScope appScope = new IndexScopeImpl( ownerId, "user" ); ApplicationEntityIndex entityIndex = eif.createApplicationEntityIndex(applicationScope); - entityIndex.initializeIndex(); + final String middleName = "middleName" + UUIDUtils.newTimeUUID(); @@ -509,7 +511,7 @@ public class EntityIndexTest extends BaseIT { IndexScope appScope = new IndexScopeImpl( ownerId, "user" ); ApplicationEntityIndex entityIndex = eif.createApplicationEntityIndex(applicationScope); - entityIndex.initializeIndex(); + entityIndex.createBatch(); // Bill has favorites as string, age as string and retirement goal as number @@ -581,9 +583,9 @@ public class EntityIndexTest extends BaseIT { Id ownerId = new SimpleId( "multivaluedtype" ); ApplicationScope applicationScope = new ApplicationScopeImpl( appId ); assertNotEquals( "cluster should be ok", Health.RED, ei.getClusterHealth() ); - assertEquals( "index should be ready", Health.GREEN, ei.getIndexHealth() ); + assertEquals("index should be ready", Health.GREEN, ei.getIndexHealth()); ApplicationEntityIndex entityIndex = eif.createApplicationEntityIndex(applicationScope); - entityIndex.initializeIndex(); + ei.refresh(); assertNotEquals( "cluster should be fine", Health.RED, ei.getIndexHealth() ); @@ -603,7 +605,7 @@ public class EntityIndexTest extends BaseIT { ApplicationEntityIndex entityIndex = eif.createApplicationEntityIndex(applicationScope); - entityIndex.initializeIndex(); + final EntityIndexBatch batch = entityIndex.createBatch(); http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/07cb9dfd/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/IndexLoadTestsIT.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/IndexLoadTestsIT.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/IndexLoadTestsIT.java index 0275e53..b03f09d 100644 --- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/IndexLoadTestsIT.java +++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/IndexLoadTestsIT.java @@ -135,10 +135,6 @@ public class IndexLoadTestsIT extends BaseIT { final IndexScope indexScope = new IndexScopeImpl( applicationId, "test" ); - final ApplicationEntityIndex appEntityIndex = entityIndexFactory.createApplicationEntityIndex( scope ); - - appEntityIndex.initializeIndex(); - batchWriteTPS = metricsFactory.getMeter( IndexLoadTestsIT.class, "write.tps" ); batchWriteTimer = metricsFactory.getTimer( IndexLoadTestsIT.class, "write.timer" );
