Merge remote-tracking branch 'origin/USERGRID-405' into USERGRID-486
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/2195e655 Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/2195e655 Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/2195e655 Branch: refs/heads/USERGRID-347 Commit: 2195e655c3227355cd5eece51a0b29ce729d6579 Parents: bab5ba7 2af7489 Author: Todd Nine <[email protected]> Authored: Fri Mar 20 15:32:09 2015 -0600 Committer: Todd Nine <[email protected]> Committed: Fri Mar 20 15:32:09 2015 -0600 ---------------------------------------------------------------------- .../batch/service/SchedulerServiceImpl.java | 17 +- .../corepersistence/CpEntityManager.java | 41 +- .../corepersistence/CpEntityManagerFactory.java | 50 +- .../corepersistence/CpManagerCache.java | 6 +- .../corepersistence/CpRelationManager.java | 19 +- .../usergrid/corepersistence/CpSetup.java | 6 +- .../usergrid/corepersistence/ManagerCache.java | 3 +- .../events/EntityDeletedHandler.java | 3 +- .../events/EntityVersionDeletedHandler.java | 4 +- .../results/FilteringLoader.java | 3 +- .../usergrid/persistence/EntityManager.java | 19 +- .../persistence/EntityManagerFactory.java | 2 - .../cassandra/EntityManagerFactoryImpl.java | 5 - .../cassandra/EntityManagerImpl.java | 20 +- .../org/apache/usergrid/CoreApplication.java | 24 +- .../java/org/apache/usergrid/CoreITSetup.java | 6 + .../org/apache/usergrid/CoreITSetupImpl.java | 27 +- .../org/apache/usergrid/TestEntityIndex.java | 27 + .../corepersistence/StaleIndexCleanupTest.java | 87 ++-- .../usergrid/persistence/CollectionIT.java | 80 +-- .../usergrid/persistence/CountingMutatorIT.java | 4 +- .../persistence/EntityConnectionsIT.java | 14 +- .../usergrid/persistence/EntityManagerIT.java | 16 +- .../org/apache/usergrid/persistence/GeoIT.java | 26 +- .../persistence/GeoQueryBooleanTest.java | 4 +- .../apache/usergrid/persistence/IndexIT.java | 14 +- .../usergrid/persistence/PathQueryIT.java | 6 +- .../PerformanceEntityRebuildIndexTest.java | 35 +- .../usergrid/persistence/PermissionsIT.java | 4 +- .../cassandra/EntityManagerFactoryImplIT.java | 4 +- .../persistence/query/CollectionIoHelper.java | 2 +- .../persistence/query/ConnectionHelper.java | 2 +- .../query/IntersectionTransitivePagingIT.java | 2 +- .../query/IntersectionUnionPagingIT.java | 2 +- .../persistence/query/IteratingQueryIT.java | 34 +- .../resources/usergrid-custom-test.properties | 1 + .../persistence/index/AliasedEntityIndex.java | 7 +- .../index/ApplicationEntityIndex.java | 50 ++ .../usergrid/persistence/index/EntityIndex.java | 46 +- .../persistence/index/EntityIndexFactory.java | 2 +- .../persistence/index/IndexIdentifier.java | 26 +- .../usergrid/persistence/index/SearchType.java | 49 ++ .../usergrid/persistence/index/SearchTypes.java | 14 +- .../persistence/index/guice/IndexModule.java | 5 +- .../impl/EsApplicationEntityIndexImpl.java | 371 ++++++++++++++ .../index/impl/EsEntityIndexBatchImpl.java | 48 +- .../index/impl/EsEntityIndexFactoryImpl.java | 27 +- .../index/impl/EsEntityIndexImpl.java | 490 ++++--------------- .../persistence/index/impl/EsQueryVistor.java | 88 ++-- .../persistence/index/impl/IndexingUtils.java | 140 +++--- .../index/impl/CorePerformanceIT.java | 0 .../impl/EntityConnectionIndexImplTest.java | 0 .../persistence/index/impl/EntityIndexTest.java | 181 +++---- .../index/impl/IndexLoadTestsIT.java | 2 +- .../usergrid/rest/AbstractContextResource.java | 102 ++-- .../org/apache/usergrid/rest/IndexResource.java | 2 +- .../rest/test/RefreshIndexResource.java | 17 +- .../resources/usergrid-custom-test.properties | 2 + .../org/apache/usergrid/ServiceApplication.java | 18 +- .../apache/usergrid/management/EmailFlowIT.java | 6 +- .../usergrid/management/OrganizationIT.java | 10 +- .../org/apache/usergrid/management/RoleIT.java | 4 +- .../cassandra/ManagementServiceIT.java | 22 +- .../management/export/ExportServiceIT.java | 12 +- .../management/importer/ImportCollectionIT.java | 15 +- .../usergrid/services/ConnectionsServiceIT.java | 10 +- .../usergrid/services/GroupServiceIT.java | 6 +- .../usergrid/services/RolesServiceIT.java | 2 +- .../usergrid/services/UsersServiceIT.java | 2 +- .../AbstractServiceNotificationIT.java | 2 +- .../apns/NotificationsServiceIT.java | 40 +- .../gcm/NotificationsServiceIT.java | 4 +- .../resources/usergrid-custom-test.properties | 2 + 73 files changed, 1302 insertions(+), 1141 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2195e655/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2195e655/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/IndexLoadTestsIT.java ---------------------------------------------------------------------- diff --cc stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/IndexLoadTestsIT.java index 057c472,623f9dd..e453a53 --- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/IndexLoadTestsIT.java +++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/IndexLoadTestsIT.java @@@ -162,80 -82,15 +162,80 @@@ public class IndexLoadTestsIT extends B final Id applicationId = new SimpleId( applicationUUID, "application" ); final ApplicationScope scope = new ApplicationScopeImpl( applicationId ); - final EntityIndex index = entityIndexFactory.createEntityIndex( scope ); + final IndexScope indexScope = new IndexScopeImpl( applicationId, "test" ); + //create our index if it doesn't exist index.initializeIndex(); - ApplicationEntityIndex applicationEntityIndex = entityIndexFactory.createApplicationEntityIndex(scope); - final Observable<Entity> createEntities = createStreamFromWorkers( applicationEntityIndex, applicationId ); ++ final Observable<Entity> createEntities = createStreamFromWorkers( index, applicationId ); + + //delay our verification for indexing to happen + final Observable<DataLoadResult> dataLoadResults = + createStreamFromWorkers( index, indexScope, uniqueIdentifier ).buffer( indexTestFig.getBufferSize() ) + //perform a delay to let ES index from our batches + .delay( indexTestFig.getValidateWait(), TimeUnit.MILLISECONDS ) + //do our search in parallel, otherwise this test will take far too long + .flatMap( entitiesToValidate -> { + return Observable.from( entitiesToValidate ).map( entityObservable -> { + + + final int workerIndex = ( int ) entityObservable.getField( FIELD_WORKER_INDEX ).getValue(); + final int ordinal = ( int ) entityObservable.getField( FIELD_ORDINAL ).getValue(); + + + final Timer.Context queryTimerContext = queryTimer.time(); + + + //execute our search + final CandidateResults results = index + .search( indexScope, SearchTypes.fromTypes( indexScope.getName() ), Query.fromQLNullSafe( + "select * where " + FIELD_WORKER_INDEX + " = " + workerIndex + " AND " + FIELD_ORDINAL + + " = " + ordinal + " AND " + FIELD_UNIQUE_IDENTIFIER + " = '" + uniqueIdentifier + + "'" ) ); + + queryTps.mark(); + queryTimerContext.stop(); + + boolean found; + + if ( !results.isEmpty() && results.get( 0 ).getId().equals( entityObservable.getId() ) ) { + found = true; + } + else { + found = false; + } + + return new EntitySearchResult( entityObservable, found ); + } ).subscribeOn( Schedulers.io() ); + }, indexTestFig.getConcurrentReadThreads() ) + + //collect all the results into a single data load result + .collect( () -> new DataLoadResult(), ( dataloadResult, entitySearchResult ) -> { + if ( entitySearchResult.found ) { + dataloadResult.success(); + return; + } + + final int ordinal = ( int ) entitySearchResult.searched.getField( FIELD_ORDINAL ).getValue(); + final int worker = ( int ) entitySearchResult.searched.getField( FIELD_WORKER_INDEX ).getValue(); + + dataloadResult.failed(); - //run them all - createEntities.toBlocking().last(); + log.error( + "Could not find entity with worker {}, ordinal {}, and Id {} after waiting {} milliseconds", + worker, ordinal, entitySearchResult.searched.getId(), indexTestFig.getValidateWait() ); + } ); + + + //wait for processing to finish + final DataLoadResult result = dataLoadResults.toBlocking().last(); + + final long expectedCount = indexTestFig.getNumberOfRecords() * indexTestFig.getNumberOfWorkers(); + + assertEquals( "Excepted to have no failures", 0, result.getFailCount() ); + + assertEquals( "Excepted to find all records", expectedCount, result.getSuccessCount() ); }
