Update of Apis and tests
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/dbfff997 Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/dbfff997 Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/dbfff997 Branch: refs/heads/USERGRID-608 Commit: dbfff9970f844f2264113fc05ee290aa3ca0ae50 Parents: 30e0e45 Author: Todd Nine <tn...@apigee.com> Authored: Mon May 11 09:27:28 2015 -0700 Committer: Todd Nine <tn...@apigee.com> Committed: Tue May 12 10:29:41 2015 -0700 ---------------------------------------------------------------------- .../corepersistence/CpEntityManagerFactory.java | 14 ------ .../corepersistence/index/ReIndexAction.java | 4 ++ .../corepersistence/index/ReIndexService.java | 11 ++-- .../index/ReIndexServiceImpl.java | 5 +- .../persistence/EntityManagerFactory.java | 2 - .../PerformanceEntityRebuildIndexTest.java | 53 ++++++-------------- 6 files changed, 27 insertions(+), 62 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/dbfff997/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java ---------------------------------------------------------------------- diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java index e796545..13b6433 100644 --- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java +++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java @@ -687,20 +687,6 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application } } - @Override - public ReIndexService.IndexResponse rebuildCollectionIndex( Optional<UUID> appId, Optional<String> collection ) { - throw new UnsupportedOperationException( "Implement me" ); -// -// EntityManager em = getEntityManager( appId ); -// -// //explicitly invoke create index, we don't know if it exists or not in ES during a rebuild. -// Application app = em.getApplication(); -// -// em.reindexCollection(po, collectionName, reverse); -// -// logger.info("\n\nRebuilt index for application {} id {} collection {}\n", -// new Object[]{app.getName(), appId, collectionName}); - } @Override public void addIndex(final String indexSuffix,final int shards,final int replicas, final String writeConsistency){ http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/dbfff997/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexAction.java ---------------------------------------------------------------------- diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexAction.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexAction.java index 086b2aa..b878246 100644 --- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexAction.java +++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexAction.java @@ -29,5 +29,9 @@ import org.apache.usergrid.persistence.collection.serialization.impl.migration.E @FunctionalInterface public interface ReIndexAction { + /** + * Index this entity with the specified scope + * @param entityIdScope + */ void index( final EntityIdScope entityIdScope ); } http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/dbfff997/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexService.java ---------------------------------------------------------------------- diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexService.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexService.java index 91409fe..e594ad3 100644 --- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexService.java +++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexService.java @@ -44,12 +44,15 @@ public interface ReIndexService { /** * Perform an index rebuild - * @param appId - * @param collection + * + * @param appId The applicationId to re-index, or all applications if absent + * @param collection The collection name to re-index. Otherwise all collections in an app will be used. + * @param cursor An optional cursor to resume processing + * @param startTimestamp The time to start indexing from. All edges >= this time will be indexed. * @return */ - IndexResponse rebuildIndex( final Optional<UUID> appId, final Optional<String> collection, final Optional<String> collectionName, final Optional<String> cursor, - final Optional<Long> startTimestamp ); + IndexResponse rebuildIndex( final Optional<UUID> appId, final Optional<String> collection, final Optional<String> cursor, + final Optional<Long> startTimestamp); http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/dbfff997/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java ---------------------------------------------------------------------- diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java index be5bcab..bd1bff9 100644 --- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java +++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java @@ -84,8 +84,7 @@ public class ReIndexServiceImpl implements ReIndexService { @Override - public IndexResponse rebuildIndex( final Optional<UUID> appId, final Optional<String> collection, - final Optional<String> collectionName, final Optional<String> cursor, + public IndexResponse rebuildIndex( final Optional<UUID> appId, final Optional<String> collection, final Optional<String> cursor, final Optional<Long> startTimestamp ) { //load our last emitted Scope if a cursor is present @@ -100,7 +99,7 @@ public class ReIndexServiceImpl implements ReIndexService { //create an observable that loads each entity and indexes it, start it running with publish final ConnectableObservable<EdgeScope> runningReIndex = - allEntityIdsObservable.getEdgesToEntities( applicationScopes, collectionName, startTimestamp ) + allEntityIdsObservable.getEdgesToEntities( applicationScopes, collection, startTimestamp ) //for each edge, create our scope and index on it .doOnNext( edge -> indexService.index( new EntityIdScope( edge.getApplicationScope(), edge.getEdge().getTargetNode() ) ) ).publish(); http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/dbfff997/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java ---------------------------------------------------------------------- diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java b/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java index b3f4b62..cd7e515 100644 --- a/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java +++ b/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java @@ -174,8 +174,6 @@ public interface EntityManagerFactory { /** For testing purposes */ public void flushEntityManagerCaches(); - ReIndexService.IndexResponse rebuildCollectionIndex( Optional<UUID> appId, Optional<String> collection ); - /** * Add a new index to the application for scale * @param suffix unique indentifier for additional index http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/dbfff997/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRebuildIndexTest.java ---------------------------------------------------------------------- diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRebuildIndexTest.java b/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRebuildIndexTest.java index df2a762..a17c925 100644 --- a/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRebuildIndexTest.java +++ b/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRebuildIndexTest.java @@ -26,6 +26,8 @@ import java.util.concurrent.TimeUnit; import com.google.common.base.Optional; import org.apache.commons.lang.RandomStringUtils; + +import org.apache.usergrid.corepersistence.index.ReIndexService; import org.apache.usergrid.persistence.index.ApplicationEntityIndex; import org.junit.After; import org.junit.Before; @@ -63,9 +65,8 @@ public class PerformanceEntityRebuildIndexTest extends AbstractCoreIT { private static final MetricRegistry registry = new MetricRegistry(); private Slf4jReporter reporter; - private static final long RUNTIME_MS = TimeUnit.SECONDS.toMillis( 10 ); + private static final int ENTITIES_TO_INDEX = 2000; - private static final long WRITE_DELAY_MS = 10; @@ -99,6 +100,8 @@ public class PerformanceEntityRebuildIndexTest extends AbstractCoreIT { final EntityManager em = setup.getEmf().getEntityManager( appId ); + final ReIndexService reIndexService = setup.getInjector().getInstance( ReIndexService.class ); + // ----------------- create a bunch of entities Map<String, Object> entityMap = new HashMap<String, Object>() {{ @@ -107,37 +110,18 @@ public class PerformanceEntityRebuildIndexTest extends AbstractCoreIT { put("key3", "Some value"); }}; -// Map<String, Object> cat1map = new HashMap<String, Object>() {{ -// put("name", "enzo"); -// put("color", "orange"); -// }}; -// Map<String, Object> cat2map = new HashMap<String, Object>() {{ -// put("name", "marquee"); -// put("color", "grey"); -// }}; -// Map<String, Object> cat3map = new HashMap<String, Object>() {{ -// put("name", "bertha"); -// put("color", "tabby"); -// }}; -// -// Entity cat1 = em.create("cat", cat1map ); -// Entity cat2 = em.create("cat", cat2map ); -// Entity cat3 = em.create("cat", cat3map ); - - final long stopTime = System.currentTimeMillis() + RUNTIME_MS; List<EntityRef> entityRefs = new ArrayList<EntityRef>(); - int entityCount = 0; int herderCount = 0; int shepardCount = 0; - while ( System.currentTimeMillis() < stopTime ) { + for (int i = 0; i < ENTITIES_TO_INDEX; i++) { final Entity entity; try { - entityMap.put("key", entityCount ); + entityMap.put("key", i ); - if ( entityCount % 2 == 0 ) { + if ( i % 2 == 0 ) { entity = em.create("catherder", entityMap); herderCount++; } else { @@ -156,15 +140,13 @@ public class PerformanceEntityRebuildIndexTest extends AbstractCoreIT { } entityRefs.add(new SimpleEntityRef( entity.getType(), entity.getUuid() ) ); - if ( entityCount % 10 == 0 ) { - logger.info("Created {} entities", entityCount ); + if ( i % 10 == 0 ) { + logger.info("Created {} entities", i ); } - entityCount++; - try { Thread.sleep( WRITE_DELAY_MS ); } catch (InterruptedException ignored ) {} } - logger.info("Created {} entities", entityCount); + logger.info("Created {} entities", ENTITIES_TO_INDEX); app.refreshIndex(); // ----------------- test that we can read them, should work fine @@ -182,7 +164,7 @@ public class PerformanceEntityRebuildIndexTest extends AbstractCoreIT { logger.debug("Reading data, should fail this time "); try { - readData( em, "testTypes", entityCount, 0 ); + readData( em, "testTypes", ENTITIES_TO_INDEX, 0 ); fail("should have failed to read data"); } catch (Exception expected) {} @@ -214,10 +196,7 @@ public class PerformanceEntityRebuildIndexTest extends AbstractCoreIT { try { -// // do it forwards - setup.getEmf().rebuildCollectionIndex(Optional.of(em.getApplicationId()), Optional.<String>of("catherders")); -// -// // and backwards, just to make sure both cases are covered + reIndexService.rebuildIndex( Optional.of( em.getApplicationId()), Optional.<String>of("catherders"), Optional.absent(), Optional.absent() ); reporter.report(); registry.remove( meterName ); @@ -269,11 +248,9 @@ public class PerformanceEntityRebuildIndexTest extends AbstractCoreIT { Entity cat2 = em.create("cat", cat2map ); Entity cat3 = em.create("cat", cat3map ); - final long stopTime = System.currentTimeMillis() + RUNTIME_MS; - List<EntityRef> entityRefs = new ArrayList<EntityRef>(); int entityCount = 0; - while ( System.currentTimeMillis() < stopTime ) { + for (int i = 0; i < ENTITIES_TO_INDEX; i++) { final Entity entity; @@ -295,8 +272,6 @@ public class PerformanceEntityRebuildIndexTest extends AbstractCoreIT { logger.info("Created {} entities", entityCount ); } - entityCount++; - try { Thread.sleep( WRITE_DELAY_MS ); } catch (InterruptedException ignored ) {} } logger.info("Created {} entities", entityCount);
