Merge branch 'two-dot-o-dev' into USERGRID-593 # Conflicts: # stack/core/pom.xml # stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java # stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java # stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java # stack/core/src/main/java/org/apache/usergrid/corepersistence/CpWalker.java # stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/ApplicationObservable.java # stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/EdgesFromSourceObservable.java # stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/EdgesToTargetObservable.java # stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java # stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java # stack/corepersistence/pom.xml # stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java # stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java # stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java # stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/Query.java # stack/pom.xml
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/ec3a710a Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/ec3a710a Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/ec3a710a Branch: refs/heads/two-dot-o-dev Commit: ec3a710adf5fd8c88e27e3c248d50c45c7164d18 Parents: 6904c9d 852c974 Author: Todd Nine <tn...@apigee.com> Authored: Fri Apr 24 09:59:57 2015 -0600 Committer: Todd Nine <tn...@apigee.com> Committed: Fri Apr 24 09:59:57 2015 -0600 ---------------------------------------------------------------------- portal/index-template.html | 2 +- portal/js/data/data-controller.js | 58 +- portal/js/data/data.html | 36 +- portal/js/global/ug-service.js | 30 +- portal/js/login/login-controller.js | 15 +- .../Usergrid.Notifications/BackgroundTask.cs | 57 + .../Usergrid.Notifications/Client/PushClient.cs | 12 +- .../Usergrid.Notifications/MainPage.xaml.cs | 12 +- .../Usergrid.Notifications/MyBackgroundTask.cs | 18 + .../Usergrid.Notifications/Package.appxmanifest | 8 + .../Usergrid.Notifications.csproj | 1 + .../Usergrid.Notifications.sln | 40 + .../main/dist/init_instance/init_rest_server.sh | 6 +- .../main/resources/usergrid-default.properties | 5 +- stack/core/pom.xml | 46 +- .../batch/service/JobSchedulerService.java | 28 +- .../batch/service/SchedulerService.java | 2 +- .../batch/service/SchedulerServiceImpl.java | 23 +- .../corepersistence/ApplicationIdCache.java | 51 + .../ApplicationIdCacheFactory.java | 43 + .../corepersistence/ApplicationIdCacheFig.java | 43 + .../corepersistence/ApplicationIdCacheImpl.java | 146 + .../usergrid/corepersistence/CoreModule.java | 137 +- .../corepersistence/CpEntityManager.java | 411 +-- .../corepersistence/CpEntityManagerFactory.java | 579 ++-- .../corepersistence/CpManagerCache.java | 65 +- .../corepersistence/CpQueryProcessor.java | 94 - .../corepersistence/CpRelationManager.java | 1842 ++--------- .../usergrid/corepersistence/CpSetup.java | 32 +- .../usergrid/corepersistence/CpWalker.java | 83 +- .../usergrid/corepersistence/GuiceFactory.java | 75 +- .../usergrid/corepersistence/ManagerCache.java | 17 +- .../corepersistence/OrgApplicationCache.java | 67 - .../OrgApplicationCacheImpl.java | 181 -- .../events/EntityDeletedHandler.java | 43 +- .../events/EntityVersionCreatedHandler.java | 22 +- .../events/EntityVersionDeletedHandler.java | 144 +- .../index/AsyncIndexProvider.java | 119 + .../index/AsyncIndexService.java | 42 + .../index/InMemoryAsyncIndexService.java | 88 + .../corepersistence/index/IndexEntityEvent.java | 75 + .../index/IndexProcessorFig.java | 86 + .../corepersistence/index/IndexService.java | 52 + .../corepersistence/index/IndexServiceImpl.java | 172 + .../corepersistence/index/ReIndexAction.java | 33 + .../corepersistence/index/ReIndexService.java | 89 + .../index/ReIndexServiceImpl.java | 129 + .../index/SQSAsyncIndexService.java | 325 ++ .../migration/AppInfoMigrationPlugin.java | 291 ++ .../migration/AppInfoVersions.java | 40 + .../migration/CoreDataVersions.java | 45 + .../migration/CoreMigration.java | 53 + .../migration/CoreMigrationPlugin.java | 64 + .../migration/EntityDataMigration.java | 147 - .../migration/EntityTypeMappingMigration.java | 69 +- .../migration/GraphShardVersionMigration.java | 152 - .../migration/MigrationModuleVersionPlugin.java | 145 + .../migration/MigrationSystemVersions.java | 43 + .../corepersistence/migration/Versions.java | 14 - .../CollectionResultsLoaderFactoryImpl.java | 6 +- .../ConnectionResultsLoaderFactoryImpl.java | 6 +- .../results/ElasticSearchQueryExecutor.java | 92 +- .../corepersistence/results/EntityVerifier.java | 2 +- .../results/FilteringLoader.java | 122 +- .../corepersistence/results/ResultsLoader.java | 2 +- .../results/ResultsLoaderFactory.java | 7 +- .../results/ResultsVerifier.java | 4 +- .../results/VersionVerifier.java | 12 +- .../rx/AllEntitiesInSystemObservable.java | 101 - .../rx/ApplicationObservable.java | 130 - .../rx/EdgesFromSourceObservable.java | 65 - .../rx/EdgesToTargetObservable.java | 65 - .../corepersistence/rx/TargetIdObservable.java | 66 - .../rx/impl/AbstractGraphVisitorImpl.java | 65 + .../rx/impl/AllApplicationsObservable.java | 43 + .../rx/impl/AllApplicationsObservableImpl.java | 133 + .../rx/impl/AllEntitiesInSystemImpl.java | 53 + .../rx/impl/AllEntityIdsObservable.java | 52 + .../rx/impl/AllEntityIdsObservableImpl.java | 92 + .../rx/impl/AllNodesInGraphImpl.java | 54 + .../corepersistence/rx/impl/EdgeScope.java | 51 + .../corepersistence/util/CpEntityMapUtils.java | 19 +- .../corepersistence/util/CpNamingUtils.java | 247 +- .../util/SerializableMapper.java | 91 + .../usergrid/exception/ConflictException.java | 36 + .../main/java/org/apache/usergrid/mq/Query.java | 2 +- .../org/apache/usergrid/mq/QueryProcessor.java | 210 +- .../apache/usergrid/mq/cassandra/QueuesCF.java | 5 - .../usergrid/persistence/CounterQuery.java | 2 +- .../usergrid/persistence/EntityManager.java | 51 +- .../persistence/EntityManagerFactory.java | 134 +- .../persistence/EntityPropertyComparator.java | 56 - .../persistence/GuiceAdapterBeanFactory.java | 114 + .../persistence/MultiQueryIterator.java | 4 +- .../persistence/ObservableIterator.java | 1 + .../persistence/PagingResultsIterator.java | 4 +- .../apache/usergrid/persistence/PathQuery.java | 6 +- .../usergrid/persistence/PersistenceModule.java | 66 + .../org/apache/usergrid/persistence/Query.java | 912 ++++++ .../apache/usergrid/persistence/QueryUtils.java | 61 +- .../usergrid/persistence/RelationManager.java | 7 +- .../apache/usergrid/persistence/Results.java | 70 +- .../org/apache/usergrid/persistence/Schema.java | 5 +- .../persistence/cassandra/ApplicationCF.java | 37 - .../cassandra/CassandraPersistenceUtils.java | 122 +- .../persistence/cassandra/CassandraService.java | 304 +- .../persistence/cassandra/CursorCache.java | 137 - .../cassandra/EntityManagerFactoryImpl.java | 485 --- .../cassandra/EntityManagerImpl.java | 2946 ------------------ .../cassandra/EntityValueSerializer.java | 53 - .../persistence/cassandra/GeoIndexManager.java | 330 -- .../persistence/cassandra/IndexUpdate.java | 451 --- .../persistence/cassandra/QueryProcessor.java | 60 - .../cassandra/QueryProcessorImpl.java | 727 ----- .../cassandra/RelationManagerImpl.java | 2298 -------------- .../persistence/cassandra/SetupImpl.java | 170 - .../cassandra/SimpleIndexBucketLocatorImpl.java | 122 - .../cassandra/index/ConnectedIndexScanner.java | 280 -- .../cassandra/index/IndexBucketScanner.java | 238 -- .../index/IndexMultiBucketSetLoader.java | 139 - .../cassandra/index/IndexScanner.java | 40 - .../cassandra/index/NoOpIndexScanner.java | 95 - .../DuplicateUniquePropertyExistsException.java | 2 +- .../persistence/geo/CollectionGeoSearch.java | 68 - .../persistence/geo/ConnectionGeoSearch.java | 67 - .../persistence/geo/EntityLocationRef.java | 227 -- .../EntityLocationRefDistanceComparator.java | 78 - .../persistence/geo/GeoIndexSearcher.java | 370 --- .../persistence/geo/GeocellManager.java | 195 -- .../usergrid/persistence/geo/GeocellUtils.java | 543 ---- .../geo/comparator/DoubleTupleComparator.java | 39 - .../persistence/geo/model/BoundingBox.java | 74 - .../persistence/geo/model/CostFunction.java | 36 - .../geo/model/DefaultCostFunction.java | 37 - .../usergrid/persistence/geo/model/Point.java | 61 - .../usergrid/persistence/geo/model/Tuple.java | 40 - .../usergrid/persistence/query/ir/AllNode.java | 82 - .../usergrid/persistence/query/ir/AndNode.java | 44 - .../persistence/query/ir/BooleanNode.java | 65 - .../query/ir/EmailIdentifierNode.java | 58 - .../query/ir/NameIdentifierNode.java | 56 - .../persistence/query/ir/NodeVisitor.java | 79 - .../usergrid/persistence/query/ir/NotNode.java | 75 - .../usergrid/persistence/query/ir/OrNode.java | 53 - .../persistence/query/ir/OrderByNode.java | 105 - .../persistence/query/ir/QueryNode.java | 41 - .../persistence/query/ir/QuerySlice.java | 345 -- .../persistence/query/ir/SearchVisitor.java | 270 -- .../persistence/query/ir/SliceNode.java | 180 -- .../query/ir/UuidIdentifierNode.java | 60 - .../persistence/query/ir/WithinNode.java | 109 - .../query/ir/result/AbstractScanColumn.java | 83 - .../result/CollectionResultsLoaderFactory.java | 41 - .../ir/result/ConnectionIndexSliceParser.java | 87 - .../query/ir/result/ConnectionRefLoader.java | 81 - .../result/ConnectionResultsLoaderFactory.java | 50 - .../ir/result/ConnectionTypesIterator.java | 190 -- .../query/ir/result/EmptyIterator.java | 63 - .../query/ir/result/EntityRefLoader.java | 51 - .../query/ir/result/EntityResultsLoader.java | 47 - .../query/ir/result/GeoIterator.java | 351 --- .../persistence/query/ir/result/IDLoader.java | 40 - .../query/ir/result/IntersectionIterator.java | 170 - .../query/ir/result/MergeIterator.java | 150 - .../query/ir/result/MultiIterator.java | 55 - .../query/ir/result/OrderByIterator.java | 250 -- .../query/ir/result/ResultIterator.java | 44 - .../query/ir/result/ResultsLoader.java | 30 - .../query/ir/result/ResultsLoaderFactory.java | 37 - .../persistence/query/ir/result/ScanColumn.java | 32 - .../query/ir/result/ScanColumnTransformer.java | 39 - .../ir/result/SecondaryIndexSliceParser.java | 61 - .../query/ir/result/SliceIterator.java | 242 -- .../query/ir/result/SliceParser.java | 32 - .../query/ir/result/StaticIdIterator.java | 82 - .../query/ir/result/SubtractionIterator.java | 113 - .../query/ir/result/UUIDIndexSliceParser.java | 47 - .../query/ir/result/UnionIterator.java | 264 -- .../main/resources/usergrid-core-context.xml | 32 +- .../java/org/apache/usergrid/Application.java | 2 +- .../org/apache/usergrid/CoreApplication.java | 37 +- .../java/org/apache/usergrid/CoreITSetup.java | 10 +- .../org/apache/usergrid/CoreITSetupImpl.java | 69 +- .../org/apache/usergrid/TestEntityIndex.java | 27 + .../usergrid/batch/job/SchedulerRuntime8IT.java | 8 +- .../corepersistence/StaleIndexCleanupTest.java | 111 +- .../corepersistence/TestIndexModule.java | 43 + .../index/AsyncIndexServiceTest.java | 203 ++ .../index/InMemoryAsycIndexServiceTest.java | 64 + .../corepersistence/index/IndexServiceTest.java | 327 ++ .../corepersistence/index/PublishRxTest.java | 95 + .../index/SQSAsyncIndexServiceTest.java | 108 + .../migration/EntityDataMigrationIT.java | 262 -- .../migration/EntityTypeMappingMigrationIT.java | 183 +- .../migration/GraphShardVersionMigrationIT.java | 226 -- .../MigrationModuleVersionPluginTest.java | 259 ++ .../migration/MigrationTestRule.java | 99 - .../migration/TestProgressObserver.java | 71 - .../rx/AllEntitiesInSystemObservableIT.java | 71 +- .../rx/ApplicationObservableTestIT.java | 17 +- .../rx/EdgesFromSourceObservableIT.java | 19 +- .../rx/EdgesToTargetObservableIT.java | 59 +- .../rx/TargetIdObservableTestIT.java | 7 +- .../usergrid/persistence/CollectionIT.java | 379 ++- .../apache/usergrid/persistence/CounterIT.java | 11 +- .../usergrid/persistence/CountingMutatorIT.java | 6 +- .../persistence/EntityConnectionsIT.java | 16 +- .../usergrid/persistence/EntityManagerIT.java | 62 +- .../EntityPropertyComparatorTest.java | 222 -- .../org/apache/usergrid/persistence/GeoIT.java | 122 +- .../persistence/GeoQueryBooleanTest.java | 24 +- .../apache/usergrid/persistence/IndexIT.java | 64 +- .../usergrid/persistence/LargeEntityIT.java | 3 +- .../usergrid/persistence/PathQueryIT.java | 27 +- .../PerformanceEntityRebuildIndexTest.java | 60 +- .../usergrid/persistence/PermissionsIT.java | 6 +- .../apache/usergrid/persistence/QueryTest.java | 328 -- .../cassandra/EntityManagerFactoryImplIT.java | 126 +- .../cassandra/QueryProcessorTest.java | 822 ----- .../SimpleIndexShardLocatorImplTest.java | 177 -- ...EntityLocationRefDistanceComparatorTest.java | 104 - .../persistence/query/CollectionIoHelper.java | 4 +- .../persistence/query/ConnectionHelper.java | 4 +- .../query/IntersectionTransitivePagingIT.java | 6 +- .../query/IntersectionUnionPagingIT.java | 6 +- .../usergrid/persistence/query/IoHelper.java | 2 +- .../persistence/query/IteratingQueryIT.java | 157 +- .../persistence/query/NotSubPropertyIT.java | 4 +- .../query/OrderByDiscardPagesPagingIT.java | 164 - .../query/ir/result/AbstractScanColumnTest.java | 101 - .../query/ir/result/InOrderIterator.java | 132 - .../ir/result/IntersectionIteratorTest.java | 308 -- .../query/ir/result/IteratorHelper.java | 34 - .../ir/result/SubtractionIteratorTest.java | 182 -- .../query/ir/result/UnionIteratorTest.java | 468 --- .../apache/usergrid/utils/EdgeTestUtils.java | 50 + stack/core/src/test/resources/log4j.properties | 4 + .../resources/usergrid-custom-test.properties | 4 +- .../resources/usergrid-properties-context.xml | 40 - .../test/resources/usergrid-test-context.xml | 1 + stack/corepersistence/collection/pom.xml | 14 +- .../persistence/collection/CollectionScope.java | 45 - .../collection/EntityCollectionManager.java | 55 +- .../EntityCollectionManagerFactory.java | 36 +- .../collection/EntityCollectionManagerSync.java | 49 - .../persistence/collection/FieldSet.java | 2 +- .../persistence/collection/MvccEntity.java | 5 +- .../persistence/collection/MvccLogEntry.java | 1 + .../cache/CachedEntityCollectionManager.java | 21 +- .../collection/event/EntityDeleted.java | 8 +- .../collection/event/EntityVersionCreated.java | 7 +- .../collection/event/EntityVersionDeleted.java | 11 +- .../exception/CollectionRuntimeException.java | 30 +- .../exception/EntityTooLargeException.java | 6 +- .../exception/WriteCommitException.java | 11 +- .../WriteOptimisticVerifyException.java | 11 +- .../exception/WriteStartException.java | 11 +- .../exception/WriteUniqueVerifyException.java | 6 +- .../collection/guice/CollectionModule.java | 48 +- .../guice/CollectionTaskExecutor.java | 4 +- .../persistence/collection/guice/Write.java | 17 - .../collection/guice/WriteUpdate.java | 17 - .../collection/impl/CollectionScopeImpl.java | 102 - .../EntityCollectionManagerFactoryImpl.java | 93 +- .../impl/EntityCollectionManagerImpl.java | 197 +- .../impl/EntityCollectionManagerSyncImpl.java | 71 - .../collection/impl/EntityDeletedTask.java | 58 +- .../impl/EntityVersionCleanupTask.java | 197 +- .../impl/EntityVersionCreatedTask.java | 39 +- .../impl/EntityVersionTaskFactory.java | 17 +- .../mvcc/MvccEntitySerializationStrategy.java | 102 - .../mvcc/MvccLogEntrySerializationStrategy.java | 80 - .../collection/mvcc/changelog/ChangeLog.java | 2 - .../mvcc/entity/MvccValidationUtils.java | 16 - .../mvcc/entity/impl/MvccEntityDeleteEvent.java | 8 +- .../mvcc/entity/impl/MvccEntityEvent.java | 11 +- .../mvcc/entity/impl/MvccEntityWriteEvent.java | 6 +- .../mvcc/stage/CollectionIoEvent.java | 9 +- .../mvcc/stage/EntityUpdateEvent.java | 34 - .../mvcc/stage/delete/MarkCommit.java | 15 +- .../collection/mvcc/stage/delete/MarkStart.java | 20 +- .../mvcc/stage/write/RollbackAction.java | 14 +- .../mvcc/stage/write/WriteCommit.java | 32 +- .../mvcc/stage/write/WriteOptimisticVerify.java | 12 +- .../collection/mvcc/stage/write/WriteStart.java | 22 +- .../mvcc/stage/write/WriteUniqueVerify.java | 45 +- .../collection/serialization/EntityRepair.java | 38 - .../MvccEntitySerializationStrategy.java | 117 + .../MvccLogEntrySerializationStrategy.java | 82 + .../serialization/OptimisticUpdate.java | 23 - .../collection/serialization/UniqueValue.java | 1 - .../UniqueValueSerializationStrategy.java | 65 +- .../serialization/UniqueValueSet.java | 1 + .../impl/CollectionDataVersions.java | 54 + .../serialization/impl/EntityRepairImpl.java | 149 - .../impl/EntityVersionSerializer.java | 11 +- .../serialization/impl/FieldSerializer.java | 118 - .../serialization/impl/LogEntryIterator.java | 8 +- .../serialization/impl/MutableFieldSet.java | 6 +- .../MvccEntitySerializationStrategyImpl.java | 177 +- ...vccEntitySerializationStrategyProxyImpl.java | 163 +- .../MvccEntitySerializationStrategyV1Impl.java | 9 +- .../MvccEntitySerializationStrategyV2Impl.java | 8 +- .../MvccEntitySerializationStrategyV3Impl.java | 574 ++++ .../MvccLogEntrySerializationProxyImpl.java | 149 + .../MvccLogEntrySerializationStrategyImpl.java | 92 +- ...MvccLogEntrySerializationStrategyV1Impl.java | 118 + ...MvccLogEntrySerializationStrategyV2Impl.java | 106 + .../serialization/impl/SerializationModule.java | 170 +- .../serialization/impl/TypeField.java | 43 + .../serialization/impl/UniqueFieldEntry.java | 60 + .../impl/UniqueFieldEntrySerializer.java | 138 + .../impl/UniqueFieldRowKeySerializer.java | 122 + .../impl/UniqueTypeFieldRowKeySerializer.java | 60 + .../serialization/impl/UniqueValueImpl.java | 5 +- .../UniqueValueSerializationStrategyImpl.java | 322 +- ...iqueValueSerializationStrategyProxyImpl.java | 179 ++ .../UniqueValueSerializationStrategyV1Impl.java | 159 + .../UniqueValueSerializationStrategyV2Impl.java | 139 + .../impl/migration/CollectionMigration.java | 53 + .../migration/CollectionMigrationPlugin.java | 68 + .../impl/migration/EntityIdScope.java | 79 + .../migration/MvccEntityDataMigrationImpl.java | 280 ++ .../serialization/impl/util/Inflector.java | 370 +++ .../impl/util/LegacyScopeUtils.java | 51 + .../collection/util/EntityUtils.java | 71 - .../collection/ApplicationContextImplTest.java | 33 + .../collection/CollectionContextImplTest.java | 53 - .../EntityCollectionManagerFactoryTest.java | 9 +- .../collection/EntityCollectionManagerIT.java | 271 +- .../EntityCollectionManagerStressTest.java | 14 +- .../EntityCollectionManagerSyncIT.java | 193 -- .../collection/guice/TestCollectionModule.java | 22 +- .../impl/EntityVersionCleanupTaskTest.java | 683 ++-- .../impl/EntityVersionCreatedTaskTest.java | 29 +- .../mvcc/stage/AbstractEntityStageTest.java | 6 +- .../mvcc/stage/AbstractIdStageTest.java | 4 +- .../mvcc/stage/AbstractMvccEntityStageTest.java | 6 +- .../mvcc/stage/TestEntityGenerator.java | 2 +- .../mvcc/stage/delete/MarkCommitTest.java | 14 +- .../mvcc/stage/delete/MarkStartTest.java | 6 +- .../mvcc/stage/write/FieldSerializerTest.java | 52 - .../write/UniqueFieldRowKeySerializerTest.java | 52 + ...niqueValueSerializationStrategyImplTest.java | 185 -- .../mvcc/stage/write/WriteCommitTest.java | 12 +- .../stage/write/WriteOptimisticVerifyTest.java | 35 +- .../mvcc/stage/write/WriteStartTest.java | 26 +- .../mvcc/stage/write/WriteUniqueVerifyIT.java | 34 +- .../mvcc/stage/write/WriteUniqueVerifyTest.java | 13 +- .../persistence/collection/rx/ParallelTest.java | 10 +- .../serialization/EntityRepairImplTest.java | 147 - .../impl/LogEntryIteratorTest.java | 17 +- ...MvccEntitySerializationStrategyImplTest.java | 165 +- ...cEntitySerializationStrategyProxyV1Test.java | 85 - ...ntitySerializationStrategyProxyV1_3Test.java | 81 + ...cEntitySerializationStrategyProxyV2Test.java | 83 - ...ntitySerializationStrategyProxyV2_3Test.java | 82 + ...ccEntitySerializationStrategyV1ImplTest.java | 78 +- ...ccEntitySerializationStrategyV2ImplTest.java | 73 +- .../MvccEntitySerializationStrategyV2Test.java | 27 +- ...ccEntitySerializationStrategyV3ImplTest.java | 70 + .../impl/MvccLESSTransientTest.java | 27 +- ...ccLogEntrySerializationStrategyImplTest.java | 107 +- ...EntrySerializationStrategyProxyImplTest.java | 90 + ...LogEntrySerializationStrategyV1ImplTest.java | 55 + ...LogEntrySerializationStrategyV2ImplTest.java | 53 + .../impl/ScopedRowKeySerializerTest.java | 2 +- .../impl/SerializationComparison.java | 4 +- .../impl/SettingsValidationTest.java | 1 - .../impl/UniqueFieldEntrySerializerTest.java | 108 + ...niqueValueSerializationStrategyImplTest.java | 344 ++ ...ValueSerializationStrategyProxyImplTest.java | 100 + ...queValueSerializationStrategyV1ImplTest.java | 57 + ...queValueSerializationStrategyV2ImplTest.java | 58 + ...ctMvccEntityDataMigrationV1ToV3ImplTest.java | 214 ++ .../MvccEntityDataMigrationV1ToV3ImplTest.java | 99 + .../MvccEntityDataMigrationV2ToV3ImplTest.java | 99 + .../collection/util/EntityHelper.java | 1 - .../collection/util/InvalidEntityGenerator.java | 1 + .../collection/util/LogEntryMock.java | 47 +- .../collection/util/UniqueValueEntryMock.java | 150 + .../collection/util/VersionGenerator.java | 55 + stack/corepersistence/common/pom.xml | 21 +- .../usergrid/persistence/core/CPManager.java | 28 + .../persistence/core/astyanax/CassandraFig.java | 3 - .../astyanax/DynamicCompositeParserImpl.java | 81 + .../core/astyanax/FieldBufferBuilder.java | 15 + .../core/astyanax/FieldBufferParser.java | 13 + .../astyanax/MultiKeyColumnNameIterator.java | 4 +- .../core/astyanax/MultiRowColumnIterator.java | 12 +- .../persistence/core/guice/CommonModule.java | 27 +- .../persistence/core/guice/CurrentImpl.java | 42 - .../persistence/core/guice/PreviousImpl.java | 42 - .../persistence/core/guice/ProxyImpl.java | 42 - .../core/hystrix/HystrixCassandra.java | 94 - .../core/metrics/MetricsFactory.java | 2 +- .../core/metrics/MetricsFactoryImpl.java | 44 +- .../core/metrics/ObservableTimer.java | 72 + .../migration/data/AbstractMigrationPlugin.java | 128 + .../core/migration/data/DataMigration.java | 97 +- .../migration/data/DataMigrationManager.java | 18 +- .../data/DataMigrationManagerImpl.java | 237 +- .../migration/data/MigrationDataProvider.java | 43 + .../core/migration/data/MigrationInfoCache.java | 52 + .../migration/data/MigrationInfoCacheImpl.java | 84 + .../data/MigrationInfoSerialization.java | 28 +- .../data/MigrationInfoSerializationImpl.java | 67 +- .../core/migration/data/MigrationPlugin.java | 56 + .../migration/data/MigrationRelationship.java | 117 + .../core/migration/data/PluginPhase.java | 42 + .../core/migration/data/ProgressObserver.java | 63 + .../core/migration/data/VersionedData.java | 38 + .../migration/data/VersionedMigrationSet.java | 164 + .../core/migration/schema/Migration.java | 2 +- .../persistence/core/rx/ObservableIterator.java | 2 + .../persistence/core/rx/RxSchedulerFig.java | 60 + .../persistence/core/rx/RxTaskScheduler.java | 40 + .../core/rx/RxTaskSchedulerImpl.java | 131 + .../core/scope/ApplicationScope.java | 9 +- .../core/scope/ApplicationScopeImpl.java | 24 +- .../persistence/core/task/TaskExecutor.java | 4 +- .../persistence/core/util/StringUtils.java | 34 + .../MultiKeyColumnNameIteratorTest.java | 187 +- .../astyanax/MultiRowColumnIteratorTest.java | 50 +- .../persistence/core/aws/NoAWSCredsRule.java | 98 + .../core/guice/DataMigrationResetRule.java | 88 + .../core/guice/MaxMigrationModule.java | 39 - .../core/guice/MaxMigrationVersion.java | 40 - .../core/guice/MigrationManagerRule.java | 7 + .../core/guice/TestCommonModule.java | 1 + .../persistence/core/guice/TestModule.java | 5 +- .../data/DataMigrationManagerImplTest.java | 350 ++- .../data/MigrationInfoSerializationTest.java | 32 +- .../data/TestMigrationDataProvider.java | 61 + .../migration/data/TestProgressObserver.java | 89 + .../data/VersionedMigrationSetTest.java | 194 ++ .../persistence/core/util/IdGenerator.java | 51 + .../persistence/graph/GraphManager.java | 3 +- .../persistence/graph/GraphManagerFactory.java | 2 + .../persistence/graph/guice/GraphModule.java | 106 +- .../graph/impl/GraphManagerImpl.java | 41 +- .../graph/impl/stage/EdgeDeleteRepairImpl.java | 9 +- .../graph/impl/stage/EdgeMetaRepairImpl.java | 21 +- .../impl/stage/NodeDeleteListenerImpl.java | 21 +- .../EdgeMetadataSerialization.java | 3 +- .../graph/serialization/EdgesObservable.java | 71 + .../graph/serialization/TargetIdObservable.java | 38 + .../EdgeMetadataSerializationProxyImpl.java | 161 +- .../impl/EdgeMetadataSerializationV1Impl.java | 6 + .../impl/EdgeMetadataSerializationV2Impl.java | 6 + .../serialization/impl/EdgesObservableImpl.java | 129 + .../serialization/impl/GraphDataVersions.java | 43 + .../impl/GraphManagerFactoryImpl.java | 95 + .../impl/NodeSerializationImpl.java | 36 +- .../impl/TargetIdObservableImpl.java | 72 + .../impl/migration/EdgeDataMigrationImpl.java | 136 + .../impl/migration/GraphMigration.java | 53 + .../impl/migration/GraphMigrationPlugin.java | 69 + .../serialization/impl/migration/GraphNode.java | 39 + .../shard/count/NodeShardApproximationImpl.java | 13 +- .../NodeShardCounterSerializationImpl.java | 25 +- .../shard/impl/NodeShardAllocationImpl.java | 16 +- .../shard/impl/ShardGroupCompactionImpl.java | 64 +- .../persistence/graph/GraphManagerIT.java | 35 +- .../persistence/graph/GraphManagerLoadTest.java | 13 +- .../graph/GraphManagerShardConsistencyIT.java | 12 +- .../graph/GraphManagerShardingIT.java | 13 +- .../graph/GraphManagerStressTest.java | 11 +- .../usergrid/persistence/graph/SimpleTest.java | 12 +- .../graph/guice/TestGraphModule.java | 27 +- .../graph/impl/EdgeDeleteListenerTest.java | 12 +- .../graph/impl/NodeDeleteListenerTest.java | 10 +- .../graph/impl/stage/EdgeDeleteRepairTest.java | 7 +- .../graph/impl/stage/EdgeMetaRepairTest.java | 30 +- .../EdgeMetaDataSerializationProxyV1Test.java | 15 +- .../EdgeMetaDataSerializationProxyV2Test.java | 23 +- .../EdgeMetaDataSerializationV1Test.java | 7 +- .../EdgeMetaDataSerializationV2Test.java | 7 +- .../EdgeMetadataSerializationTest.java | 39 +- .../EdgeSerializationChopTest.java | 7 +- .../serialization/EdgeSerializationTest.java | 21 +- .../serialization/NodeSerializationTest.java | 15 +- .../migration/EdgeDataMigrationImplTest.java | 177 ++ .../impl/shard/EdgeShardSerializationTest.java | 7 +- .../impl/shard/NodeShardAllocationTest.java | 27 +- .../impl/shard/NodeShardCacheTest.java | 7 +- .../impl/shard/ShardGroupCompactionTest.java | 7 +- .../shard/count/NodeShardApproximationTest.java | 9 +- .../NodeShardCounterSerializationTest.java | 6 +- .../shard/impl/ShardEntryGroupIteratorTest.java | 15 +- ...rceDirectedEdgeDescendingComparatorTest.java | 23 +- ...getDirectedEdgeDescendingComparatorTest.java | 23 +- .../graph/test/util/EdgeTestUtils.java | 29 +- .../usergrid/persistence/map/MapManager.java | 25 +- .../persistence/map/guice/MapModule.java | 8 +- .../persistence/map/guice/TestMapModule.java | 3 +- stack/corepersistence/model/pom.xml | 1 - .../persistence/model/entity/Entity.java | 23 +- .../persistence/model/entity/EntityMap.java | 80 + .../model/entity/EntityToMapConverter.java | 160 + .../model/entity/MapToEntityConverter.java | 203 ++ .../persistence/model/entity/SimpleId.java | 36 +- .../persistence/model/field/AbstractField.java | 8 + .../persistence/model/field/ListField.java | 8 +- .../persistence/model/field/StringField.java | 20 + .../persistence/model/util/EntityUtils.java | 72 + stack/corepersistence/pom.xml | 36 +- stack/corepersistence/queryindex/pom.xml | 63 +- .../index/query/tree/CpQueryFilter.g | 20 +- .../persistence/index/AliasedEntityIndex.java | 13 +- .../index/ApplicationEntityIndex.java | 55 + .../persistence/index/CandidateResult.java | 84 + .../persistence/index/CandidateResults.java | 106 + .../usergrid/persistence/index/EntityIndex.java | 75 +- .../persistence/index/EntityIndexBatch.java | 25 +- .../persistence/index/EntityIndexFactory.java | 11 +- .../persistence/index/IndexBatchBuffer.java | 36 - .../persistence/index/IndexBufferConsumer.java | 37 - .../persistence/index/IndexBufferProducer.java | 37 - .../usergrid/persistence/index/IndexEdge.java | 36 + .../usergrid/persistence/index/IndexFig.java | 128 +- .../persistence/index/IndexIdentifier.java | 92 - .../index/IndexOperationMessage.java | 138 - .../persistence/index/IndexRefreshCommand.java | 50 + .../usergrid/persistence/index/IndexScope.java | 38 - .../usergrid/persistence/index/SearchEdge.java | 61 + .../usergrid/persistence/index/SearchType.java | 51 + .../usergrid/persistence/index/SearchTypes.java | 46 +- .../persistence/index/SelectFieldMapping.java | 58 + .../persistence/index/guice/IndexModule.java | 44 +- .../persistence/index/guice/QueueProvider.java | 116 - .../persistence/index/impl/BatchOperation.java | 41 + .../persistence/index/impl/BatchRequest.java | 41 - .../persistence/index/impl/BufferQueue.java | 68 - .../index/impl/BufferQueueInMemoryImpl.java | 116 - .../index/impl/BufferQueueSQSImpl.java | 307 -- .../index/impl/DeIndexOperation.java | 104 + .../persistence/index/impl/DeIndexRequest.java | 115 - .../persistence/index/impl/EntityField.java | 148 + .../index/impl/EntityMappingParser.java | 247 ++ .../index/impl/EntityToMapConverter.java | 108 + .../impl/EsApplicationEntityIndexImpl.java | 274 ++ .../index/impl/EsEntityIndexBatchImpl.java | 283 +- .../index/impl/EsEntityIndexFactoryImpl.java | 44 +- .../index/impl/EsEntityIndexImpl.java | 590 +--- .../index/impl/EsIndexBufferConsumerImpl.java | 348 +-- .../index/impl/EsIndexBufferProducerImpl.java | 65 - .../persistence/index/impl/EsIndexCache.java | 142 - .../index/impl/EsIndexCacheImpl.java | 141 + .../persistence/index/impl/EsProvider.java | 129 +- .../persistence/index/impl/EsQueryVistor.java | 589 ++-- .../persistence/index/impl/FieldParser.java | 45 + .../index/impl/FlushBufferQueue.java | 23 + .../persistence/index/impl/GeoSortFields.java | 95 + .../persistence/index/impl/IndexAlias.java | 42 + .../index/impl/IndexBufferConsumer.java | 38 + .../persistence/index/impl/IndexCache.java | 43 + .../persistence/index/impl/IndexEdgeImpl.java | 84 + .../persistence/index/impl/IndexIdentifier.java | 46 + .../index/impl/IndexIdentifierImpl.java | 67 + .../persistence/index/impl/IndexOperation.java | 107 + .../index/impl/IndexOperationMessage.java | 126 + .../index/impl/IndexRefreshCommandImpl.java | 170 + .../persistence/index/impl/IndexRequest.java | 125 - .../persistence/index/impl/IndexScopeImpl.java | 88 - .../persistence/index/impl/IndexingUtils.java | 267 +- .../index/impl/NoOpFilterBuilder.java | 45 + .../index/impl/NoOpQueryBuilder.java | 45 + .../persistence/index/impl/SearchEdgeImpl.java | 98 + .../impl/SearchRequestBuilderStrategy.java | 290 ++ .../persistence/index/impl/SortBuilder.java | 44 + .../index/migration/EsIndexMigrationPlugin.java | 46 + .../index/migration/IndexDataVersions.java | 39 + .../index/migration/IndexMigration.java | 35 + .../index/migration/LegacyIndexIdentifier.java | 78 + .../index/query/CandidateResult.java | 74 - .../index/query/CandidateResults.java | 90 - .../persistence/index/query/EntityResults.java | 108 - .../persistence/index/query/ParsedQuery.java | 205 ++ .../index/query/ParsedQueryBuilder.java | 108 + .../usergrid/persistence/index/query/Query.java | 1436 --------- .../persistence/index/query/Results.java | 148 - .../persistence/index/query/SortPredicate.java | 144 + .../index/query/tree/QueryVisitor.java | 55 +- .../index/utils/IndexValidationUtils.java | 24 +- .../persistence/index/utils/ListUtils.java | 1 + .../persistence/index/utils/StringUtils.java | 62 - .../persistence/index/usergrid-mappings.json | 93 + .../persistence/index/guice/IndexTestFig.java | 62 + .../index/guice/TestIndexModule.java | 42 +- .../index/impl/BufferQueueSQSImplTest.java | 169 - .../index/impl/CorePerformanceIT.java | 339 -- .../index/impl/ElasticSearchRule.java | 58 + .../impl/EntityConnectionIndexImplTest.java | 306 -- .../index/impl/EntityIndexMapUtils.java | 52 +- .../persistence/index/impl/EntityIndexTest.java | 1073 +++++-- .../index/impl/EntityToMapConverterTest.java | 653 ++++ .../persistence/index/impl/GeoPagingTest.java | 163 + .../index/impl/IndexLoadTestsIT.java | 393 +++ .../persistence/query/tree/GrammarTreeTest.java | 157 +- .../src/test/resources/dynamic-test.properties | 2 +- .../src/test/resources/log4j.properties | 10 +- .../src/test/resources/usergrid-UNIT.properties | 3 + .../usergrid/persistence/queue/QueueFig.java | 8 + .../persistence/queue/NoAWSCredsRule.java | 98 - .../persistence/queue/QueueManagerTest.java | 3 +- .../queue/guice/TestQueueModule.java | 3 +- stack/java-sdk-old/pom.xml | 102 - .../usergrid/mongo/protocol/OpDelete.java | 4 +- .../apache/usergrid/mongo/protocol/OpQuery.java | 6 +- .../usergrid/mongo/protocol/OpUpdate.java | 4 +- .../apache/usergrid/mongo/BasicMongoTest.java | 8 +- .../apache/usergrid/mongo/MongoQueryTest.java | 18 +- stack/pom.xml | 28 +- stack/rest/pom.xml | 17 +- .../usergrid/rest/AbstractContextResource.java | 102 +- .../org/apache/usergrid/rest/ApiResponse.java | 50 +- .../org/apache/usergrid/rest/IndexResource.java | 9 +- .../apache/usergrid/rest/MigrateResource.java | 122 +- .../org/apache/usergrid/rest/RootResource.java | 19 +- .../rest/applications/ApplicationResource.java | 38 +- .../rest/applications/ServiceResource.java | 43 +- .../notifiers/NotifiersResource.java | 11 +- .../rest/applications/users/UsersResource.java | 4 +- .../exceptions/AbstractExceptionMapper.java | 11 +- .../exceptions/ConflictExceptionMapper.java | 35 + .../exceptions/NotFoundExceptionMapper.java | 35 + .../rest/exceptions/UncaughtException.java | 36 + .../rest/management/ManagementResource.java | 59 +- .../organizations/OrganizationResource.java | 28 +- .../organizations/OrganizationsResource.java | 23 +- .../applications/ApplicationResource.java | 61 +- .../applications/ApplicationsResource.java | 34 +- .../imports/FileIncludesResource.java | 9 - .../applications/imports/ImportsResource.java | 83 +- .../organizations/OrganizationResource.java | 32 +- .../OAuth2AccessTokenSecurityFilter.java | 2 +- .../rest/test/RefreshIndexResource.java | 25 +- .../java/org/apache/usergrid/rest/BasicIT.java | 302 +- .../apache/usergrid/rest/IndexResourceIT.java | 1 + .../apache/usergrid/rest/NotificationsIT.java | 93 +- .../apache/usergrid/rest/SystemResourceIT.java | 10 +- .../rest/applications/ApplicationCreateIT.java | 135 + .../rest/applications/ApplicationDeleteIT.java | 389 +++ .../applications/ApplicationDeleteTest.java | 54 - .../applications/ApplicationResourceIT.java | 100 +- .../collection/CollectionsResourceIT.java | 19 +- .../collection/paging/PagingResourceIT.java | 17 +- .../collection/users/OwnershipResourceIT.java | 4 +- .../collection/users/PermissionsResourceIT.java | 15 - .../collection/users/RetrieveUsersTest.java | 4 +- .../collection/users/UserResourceIT.java | 16 +- .../events/ApplicationRequestCounterIT.java | 2 +- .../applications/queries/BasicGeoTests.java | 21 +- .../applications/queries/GeoPagingTest.java | 58 +- .../applications/queries/MatrixQueryTests.java | 298 +- .../rest/applications/queries/OrderByTest.java | 2 +- .../queues/AbstractQueueResourceIT.java | 484 --- .../queues/QueueResourceLong1IT.java | 149 - .../queues/QueueResourceLong2IT.java | 99 - .../queues/QueueResourceLong3IT.java | 201 -- .../queues/QueueResourceShortIT.java | 432 --- .../rest/filters/ContentTypeResourceIT.java | 55 +- .../usergrid/rest/management/AccessTokenIT.java | 418 +-- .../usergrid/rest/management/AdminUsersIT.java | 64 +- .../rest/management/ExportResourceIT.java | 674 ++-- .../rest/management/ImportResourceIT.java | 179 +- .../rest/management/ManagementResourceIT.java | 151 +- .../rest/management/OrganizationsIT.java | 82 +- .../organizations/AdminEmailEncodingIT.java | 3 +- .../test/resource2point0/AbstractRestIT.java | 4 +- .../rest/test/resource2point0/ClientSetup.java | 30 +- .../rest/test/resource2point0/README.md | 111 + .../rest/test/resource2point0/RestClient.java | 1 + .../endpoints/ApplicationsResource.java | 34 +- .../endpoints/CollectionEndpoint.java | 13 + .../endpoints/NamedResource.java | 158 +- .../endpoints/OrganizationResource.java | 6 +- .../resource2point0/endpoints/RootResource.java | 1 + .../endpoints/SetupResource.java | 6 + .../endpoints/mgmt/ApplicationResource.java | 54 +- .../endpoints/mgmt/ApplicationsResource.java | 58 + .../endpoints/mgmt/ManagementResource.java | 4 + .../endpoints/mgmt/ManagementResponse.java | 72 + .../endpoints/mgmt/MeResource.java | 32 + .../mgmt/OrganizationApplicationResponse.java | 60 + .../endpoints/mgmt/OrganizationResource.java | 5 +- .../endpoints/mgmt/PasswordResource.java | 16 - .../endpoints/mgmt/RevokeTokenResource.java | 32 + .../endpoints/mgmt/RevokeTokensResource.java | 33 + .../endpoints/mgmt/TokenResource.java | 25 +- .../endpoints/mgmt/UserResource.java | 8 + .../endpoints/mgmt/UsersResource.java | 15 - stack/rest/src/test/resources/log4j.properties | 10 +- .../resources/testImportCorrect.testCol.1.json | 18 - ...testImportInvalidJson.testApplication.3.json | 153 - .../testimport-bad-json-testapp.3.json | 153 + .../src/test/resources/testimport-bad-json.json | 67 + .../resources/testimport-correct-testcol.1.json | 18 + .../test/resources/testimport-test-app.2.json | 56 + .../resources/usergrid-custom-test.properties | 2 + stack/services/pom.xml | 51 +- .../usergrid/management/ApplicationInfo.java | 4 + .../usergrid/management/ManagementService.java | 10 +- .../cassandra/ApplicationCreatorImpl.java | 12 +- .../cassandra/ManagementServiceImpl.java | 236 +- .../management/export/ExportServiceImpl.java | 4 +- .../management/importer/FileImportJob.java | 8 +- .../management/importer/FileImportTracker.java | 6 +- .../usergrid/management/importer/ImportJob.java | 4 +- .../management/importer/ImportServiceImpl.java | 88 +- .../usergrid/security/AuthPrincipalType.java | 3 +- .../security/providers/AbstractProvider.java | 1 + .../security/providers/FacebookProvider.java | 6 +- .../security/providers/FoursquareProvider.java | 6 +- .../shiro/PrincipalCredentialsToken.java | 20 +- .../apache/usergrid/security/shiro/Realm.java | 2 +- .../services/AbstractCollectionService.java | 17 +- .../services/AbstractConnectionsService.java | 4 +- .../AbstractPathBasedColllectionService.java | 2 +- .../usergrid/services/AbstractService.java | 2 +- .../usergrid/services/ServiceContext.java | 2 +- .../services/ServiceManagerFactory.java | 4 - .../usergrid/services/ServiceParameter.java | 3 +- .../usergrid/services/ServiceRequest.java | 2 +- .../usergrid/services/ServiceResults.java | 38 +- .../applications/ApplicationsService.java | 2 +- .../services/devices/DevicesService.java | 2 +- .../usergrid/services/groups/GroupsService.java | 2 +- .../groups/activities/ActivitiesService.java | 2 +- .../users/activities/ActivitiesService.java | 4 +- .../notifications/InactiveDeviceManager.java | 8 +- .../notifications/NotificationsService.java | 4 +- .../apns/FailedConnectionListener.java | 31 +- .../impl/ApplicationQueueManagerImpl.java | 197 +- .../services/notifications/wns/WNSAdapter.java | 60 +- .../usergrid/services/roles/RolesService.java | 10 +- .../usergrid/services/users/UsersService.java | 2 +- .../users/activities/ActivitiesService.java | 4 +- .../org/apache/usergrid/ServiceApplication.java | 18 +- .../org/apache/usergrid/ServiceITSetup.java | 5 + .../org/apache/usergrid/ServiceITSetupImpl.java | 43 +- .../migration/AppInfoMigrationPluginTest.java | 231 ++ .../apache/usergrid/management/EmailFlowIT.java | 9 +- .../usergrid/management/OrganizationIT.java | 16 +- .../org/apache/usergrid/management/RoleIT.java | 4 +- .../cassandra/ApplicationCreatorIT.java | 14 +- .../cassandra/ManagementServiceIT.java | 35 +- .../management/export/ExportServiceIT.java | 20 +- .../importer/FileImportTrackerTest.java | 12 +- .../management/importer/ImportCollectionIT.java | 20 +- .../importer/ImportConnectionsTest.java | 9 +- .../management/importer/ImportServiceIT.java | 2 +- .../security/tokens/TokenServiceIT.java | 2 - .../usergrid/services/ActivitiesServiceIT.java | 2 +- .../usergrid/services/CollectionServiceIT.java | 2 + .../usergrid/services/ConnectionsServiceIT.java | 10 +- .../usergrid/services/GroupServiceIT.java | 6 +- .../usergrid/services/RolesServiceIT.java | 4 +- .../usergrid/services/ServiceFactoryIT.java | 8 +- .../usergrid/services/ServiceInvocationIT.java | 2 +- .../usergrid/services/ServiceRequestIT.java | 9 +- .../usergrid/services/UsersServiceIT.java | 2 +- .../AbstractServiceNotificationIT.java | 4 +- .../notifications/NotifiersServiceIT.java | 19 +- .../apns/NotificationsServiceIT.java | 66 +- .../gcm/NotificationsServiceIT.java | 4 +- .../src/test/resources/log4j.properties | 6 +- .../resources/usergrid-custom-test.properties | 2 + .../usergrid/cassandra/SpringResource.java | 13 +- .../setup/ConcurrentProcessSingleton.java | 49 +- .../org/apache/usergrid/tools/AppAudit.java | 3 +- .../org/apache/usergrid/tools/CredExport.java | 2 +- .../org/apache/usergrid/tools/DupOrgRepair.java | 1 - .../java/org/apache/usergrid/tools/Import.java | 14 +- .../usergrid/tools/UniqueIndexCleanup.java | 6 +- 776 files changed, 30517 insertions(+), 39729 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ec3a710a/stack/core/pom.xml ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ec3a710a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java ---------------------------------------------------------------------- diff --cc stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java index 038be44,23514c8..748a069 --- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java +++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java @@@ -70,11 -63,9 +63,10 @@@ import org.apache.usergrid.persistence. import org.apache.usergrid.persistence.cassandra.CassandraService; import org.apache.usergrid.persistence.cassandra.ConnectionRefImpl; import org.apache.usergrid.persistence.cassandra.CounterUtils; - import org.apache.usergrid.persistence.cassandra.GeoIndexManager; import org.apache.usergrid.persistence.cassandra.util.TraceParticipant; - import org.apache.usergrid.persistence.collection.CollectionScope; import org.apache.usergrid.persistence.collection.EntityCollectionManager; - import org.apache.usergrid.persistence.collection.exception.WriteOptimisticVerifyException; ++import org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory; + import org.apache.usergrid.persistence.collection.FieldSet; import org.apache.usergrid.persistence.collection.exception.WriteUniqueVerifyException; import org.apache.usergrid.persistence.core.metrics.MetricsFactory; import org.apache.usergrid.persistence.core.scope.ApplicationScope; @@@ -88,15 -78,8 +79,9 @@@ import org.apache.usergrid.persistence. import org.apache.usergrid.persistence.exceptions.EntityNotFoundException; import org.apache.usergrid.persistence.exceptions.RequiredPropertyNotFoundException; import org.apache.usergrid.persistence.exceptions.UnexpectedEntityTypeException; +import org.apache.usergrid.persistence.graph.GraphManagerFactory; - import org.apache.usergrid.persistence.index.EntityIndex; - import org.apache.usergrid.persistence.index.EntityIndexBatch; - import org.apache.usergrid.persistence.index.IndexScope; - import org.apache.usergrid.persistence.index.impl.IndexScopeImpl; import org.apache.usergrid.persistence.index.query.CounterResolution; import org.apache.usergrid.persistence.index.query.Identifier; - import org.apache.usergrid.persistence.index.query.Query; - import org.apache.usergrid.persistence.index.query.Query.Level; import org.apache.usergrid.persistence.map.MapManager; import org.apache.usergrid.persistence.map.MapScope; import org.apache.usergrid.persistence.model.entity.Id; @@@ -178,21 -163,20 +165,24 @@@ public class CpEntityManager implement public static final String APPLICATION_ENTITIES = "application.entities"; public static final long ONE_COUNT = 1L; - private UUID applicationId; + private final UUID applicationId; private Application application; - private ManagerCache managerCache; - private ApplicationScope applicationScope; + private final ManagerCache managerCache; + + private final ApplicationScope applicationScope; - private CassandraService cass; + private final CassandraService cass; - private CounterUtils counterUtils; + private final CounterUtils counterUtils; + + private final AsyncIndexService indexService; + + private EntityCollectionManagerFactory entityCollectionManagerFactory; + private GraphManagerFactory graphManagerFactory; + private boolean skipAggregateCounters; private MetricsFactory metricsFactory; private Timer aggCounterTimer; @@@ -220,14 -207,22 +213,28 @@@ // private LoadingCache<EntityScope, org.apache.usergrid.persistence.model.entity.Entity> entityCache; - public CpEntityManager() { - - } - - @Override - public void init( final CassandraService cassandraService, final CounterUtils counterUtils, final MetricsFactory metricsFactory, final GraphManagerFactory graphManagerFactory, final EntityCollectionManagerFactory entityCollectionManagerFactory, final ManagerCache managerCache, UUID applicationId ) { + /** + * Fugly, make this part of DI + * @param cass + * @param counterUtils + * @param managerCache + * @param metricsFactory + * @param applicationId ++ * @param entityCollectionManagerFactory ++ * @param graphManagerFactory + */ - public CpEntityManager(final CassandraService cass, final CounterUtils counterUtils, final AsyncIndexService indexService, final ManagerCache managerCache, final MetricsFactory metricsFactory, final UUID applicationId ) { ++ public CpEntityManager( final CassandraService cass, final CounterUtils counterUtils, final AsyncIndexService indexService, final ManagerCache managerCache, ++ final MetricsFactory metricsFactory,final EntityCollectionManagerFactory entityCollectionManagerFactory, ++ final GraphManagerFactory graphManagerFactory , final UUID applicationId ) { ++ this.entityCollectionManagerFactory = entityCollectionManagerFactory; ++ this.graphManagerFactory = graphManagerFactory; + Preconditions.checkNotNull( cass, "cass must not be null" ); + Preconditions.checkNotNull( counterUtils, "counterUtils must not be null" ); + Preconditions.checkNotNull( managerCache, "managerCache must not be null" ); Preconditions.checkNotNull( applicationId, "applicationId must not be null" ); + Preconditions.checkNotNull( indexService, "indexService must not be null" ); + this.managerCache = managerCache; this.applicationId = applicationId; @@@ -772,9 -733,10 +745,10 @@@ @Override public RelationManager getRelationManager( EntityRef entityRef ) { Preconditions.checkNotNull( entityRef, "entityRef cannot be null" ); - CpRelationManager rmi = new CpRelationManager(); - rmi.init( this, applicationId, entityRef, null, metricsFactory, managerCache, entityCollectionManagerFactory, graphManagerFactory ); - return rmi; + + CpRelationManager relationManager = - new CpRelationManager( metricsFactory, managerCache, indexService, this, applicationId, entityRef ); ++ new CpRelationManager( metricsFactory, managerCache, entityCollectionManagerFactory, graphManagerFactory, indexService, this, applicationId, entityRef ); + return relationManager; } @@@ -2888,34 -2818,8 +2830,7 @@@ } - @Override - public void refreshIndex() { - - // refresh factory indexes - - - // refresh this Entity Manager's application's index - EntityIndex ei = managerCache.getEntityIndex( getApplicationScope() ); - ei.refresh(); - } - - - @Override - public void createIndex() { - EntityIndex ei = managerCache.getEntityIndex( applicationScope ); - ei.initializeIndex(); - } - - public void deleteIndex(){ - EntityIndex ei = managerCache.getEntityIndex( applicationScope ); - ei.deleteIndex(); - } - - - - - - @Override public void flushManagerCaches() { managerCache.invalidate(); } http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ec3a710a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java ---------------------------------------------------------------------- diff --cc stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java index 5b8e715,f08bce4..6ba71e2 --- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java +++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java @@@ -177,10 -192,7 +192,8 @@@ public class CpEntityManagerFactory imp private EntityManager _getEntityManager( UUID applicationId ) { - - EntityManager em = new CpEntityManager(); - em.init( cassandraService, counterUtils, metricsFactory, graphManagerFactory, entityCollectionManagerFactory, managerCache, applicationId ); - - EntityManager em = new CpEntityManager(cassandraService, counterUtils, indexService, managerCache, metricsFactory, applicationId ); ++ EntityManager em = new CpEntityManager(cassandraService, counterUtils, indexService, managerCache, metricsFactory, applicationId, ++ entityCollectionManagerFactory, graphManagerFactory ); return em; } @@@ -432,17 -436,17 +437,17 @@@ logger.debug("getApplications(): Loading edges of edgeType {} from {}:{}", new Object[] { edgeType, fromEntityId.getType(), fromEntityId.getUuid() } ); - Observable<Edge> edges = gm.loadEdgesFromSource(new SimpleSearchByEdgeType( - fromEntityId, edgeType, Long.MAX_VALUE, - SearchByEdgeType.Order.DESCENDING, null)); + Observable<Edge> edges = gm.loadEdgesFromSource( new SimpleSearchByEdgeType( + fromEntityId, edgeType, Long.MAX_VALUE, + SearchByEdgeType.Order.DESCENDING, Optional.<Edge>absent() )); - Iterator<Edge> iter = edges.toBlockingObservable().getIterator(); - while ( iter.hasNext() ) { + // TODO This is wrong, and will result in OOM if there are too many applications. + // This needs to stream properly with a buffer - Edge edge = iter.next(); + edges.doOnNext(edge -> { Id targetId = edge.getTargetNode(); - logger.debug("getApplications(): Processing edge from {}:{} to {}:{}", new Object[] { + logger.debug("getApplications(): Processing edge from {}:{} to {}:{}", new Object[]{ edge.getSourceNode().getType(), edge.getSourceNode().getUuid(), edge.getTargetNode().getType(), edge.getTargetNode().getUuid() }); http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ec3a710a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java ---------------------------------------------------------------------- diff --cc stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java index edd352e,ea554a3..8e2fdd7 --- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java +++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java @@@ -54,86 -50,44 +50,48 @@@ import org.apache.usergrid.persistence. import org.apache.usergrid.persistence.Schema; import org.apache.usergrid.persistence.SimpleEntityRef; import org.apache.usergrid.persistence.SimpleRoleRef; - import org.apache.usergrid.persistence.cassandra.CassandraService; import org.apache.usergrid.persistence.cassandra.ConnectionRefImpl; - import org.apache.usergrid.persistence.cassandra.IndexUpdate; - import org.apache.usergrid.persistence.cassandra.QueryProcessorImpl; - import org.apache.usergrid.persistence.cassandra.index.ConnectedIndexScanner; - import org.apache.usergrid.persistence.cassandra.index.IndexBucketScanner; - import org.apache.usergrid.persistence.cassandra.index.IndexScanner; - import org.apache.usergrid.persistence.cassandra.index.NoOpIndexScanner; - import org.apache.usergrid.persistence.collection.CollectionScope; +import org.apache.usergrid.persistence.collection.EntityCollectionManager; +import org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory; - import org.apache.usergrid.persistence.core.future.BetterFuture; import org.apache.usergrid.persistence.core.metrics.MetricsFactory; import org.apache.usergrid.persistence.core.scope.ApplicationScope; import org.apache.usergrid.persistence.entities.Group; import org.apache.usergrid.persistence.entities.User; - import org.apache.usergrid.persistence.geo.ConnectionGeoSearch; - import org.apache.usergrid.persistence.geo.EntityLocationRef; - import org.apache.usergrid.persistence.geo.model.Point; import org.apache.usergrid.persistence.graph.Edge; import org.apache.usergrid.persistence.graph.GraphManager; +import org.apache.usergrid.persistence.graph.GraphManagerFactory; + import org.apache.usergrid.persistence.graph.SearchByEdge; import org.apache.usergrid.persistence.graph.SearchByEdgeType; import org.apache.usergrid.persistence.graph.impl.SimpleEdge; import org.apache.usergrid.persistence.graph.impl.SimpleSearchByEdge; import org.apache.usergrid.persistence.graph.impl.SimpleSearchByEdgeType; import org.apache.usergrid.persistence.graph.impl.SimpleSearchEdgeType; - import org.apache.usergrid.persistence.index.EntityIndex; + import org.apache.usergrid.persistence.index.ApplicationEntityIndex; import org.apache.usergrid.persistence.index.EntityIndexBatch; - import org.apache.usergrid.persistence.index.IndexScope; + import org.apache.usergrid.persistence.index.IndexEdge; + import org.apache.usergrid.persistence.index.SearchEdge; import org.apache.usergrid.persistence.index.SearchTypes; - import org.apache.usergrid.persistence.index.impl.IndexScopeImpl; import org.apache.usergrid.persistence.index.query.Identifier; - import org.apache.usergrid.persistence.index.query.Query; - import org.apache.usergrid.persistence.index.query.Query.Level; import org.apache.usergrid.persistence.model.entity.Id; import org.apache.usergrid.persistence.model.entity.SimpleId; - import org.apache.usergrid.persistence.model.util.UUIDGenerator; - import org.apache.usergrid.persistence.query.ir.AllNode; - import org.apache.usergrid.persistence.query.ir.NameIdentifierNode; - import org.apache.usergrid.persistence.query.ir.QueryNode; - import org.apache.usergrid.persistence.query.ir.QuerySlice; - import org.apache.usergrid.persistence.query.ir.SearchVisitor; - import org.apache.usergrid.persistence.query.ir.WithinNode; - import org.apache.usergrid.persistence.query.ir.result.ConnectionIndexSliceParser; - import org.apache.usergrid.persistence.query.ir.result.ConnectionResultsLoaderFactory; - import org.apache.usergrid.persistence.query.ir.result.ConnectionTypesIterator; - import org.apache.usergrid.persistence.query.ir.result.EmptyIterator; - import org.apache.usergrid.persistence.query.ir.result.GeoIterator; - import org.apache.usergrid.persistence.query.ir.result.SliceIterator; - import org.apache.usergrid.persistence.query.ir.result.StaticIdIterator; import org.apache.usergrid.persistence.schema.CollectionInfo; - import org.apache.usergrid.utils.IndexUtils; import org.apache.usergrid.utils.MapUtils; - import org.apache.usergrid.utils.UUIDUtils; import com.codahale.metrics.Timer; +import com.google.common.base.Optional; import com.google.common.base.Preconditions; - import me.prettyprint.hector.api.Keyspace; - import me.prettyprint.hector.api.beans.DynamicComposite; - import me.prettyprint.hector.api.beans.HColumn; - import me.prettyprint.hector.api.mutation.Mutator; import rx.Observable; - import rx.functions.Action1; - import rx.functions.Action2; import rx.functions.Func1; - import static java.util.Arrays.asList; - - import static me.prettyprint.hector.api.factory.HFactory.createMutator; - import static org.apache.usergrid.corepersistence.util.CpNamingUtils.getCollectionScopeNameFromEntityType; + import static org.apache.usergrid.corepersistence.util.CpNamingUtils.createCollectionEdge; + import static org.apache.usergrid.corepersistence.util.CpNamingUtils.createCollectionSearchEdge; + import static org.apache.usergrid.corepersistence.util.CpNamingUtils.createConnectionSearchByEdge; + import static org.apache.usergrid.corepersistence.util.CpNamingUtils.createConnectionSearchEdge; + import static org.apache.usergrid.corepersistence.util.CpNamingUtils.createSearchEdgeFromSource; + import static org.apache.usergrid.corepersistence.util.CpNamingUtils.generateScopeFromSource; + import static org.apache.usergrid.corepersistence.util.CpNamingUtils.getNameFromEdgeType; import static org.apache.usergrid.persistence.Schema.COLLECTION_ROLES; - import static org.apache.usergrid.persistence.Schema.DICTIONARY_CONNECTED_ENTITIES; - import static org.apache.usergrid.persistence.Schema.DICTIONARY_CONNECTED_TYPES; - import static org.apache.usergrid.persistence.Schema.DICTIONARY_CONNECTING_ENTITIES; - import static org.apache.usergrid.persistence.Schema.DICTIONARY_CONNECTING_TYPES; - import static org.apache.usergrid.persistence.Schema.INDEX_CONNECTIONS; - import static org.apache.usergrid.persistence.Schema.PROPERTY_CREATED; import static org.apache.usergrid.persistence.Schema.PROPERTY_INACTIVITY; import static org.apache.usergrid.persistence.Schema.PROPERTY_NAME; import static org.apache.usergrid.persistence.Schema.PROPERTY_TITLE; @@@ -172,10 -106,7 +110,9 @@@ public class CpRelationManager implemen private static final Logger logger = LoggerFactory.getLogger( CpRelationManager.class ); - private ManagerCache managerCache; + private EntityCollectionManagerFactory entityCollectionManagerFactory; + private GraphManagerFactory graphManagerFactory; private EntityManager em; @@@ -194,22 -122,10 +128,11 @@@ private MetricsFactory metricsFactory; private Timer updateCollectionTimer; - private Timer createConnectionTimer; - private Timer cassConnectionDelete; - private Timer esDeleteConnectionTimer; - public CpRelationManager() {} - public CpRelationManager( final MetricsFactory metricsFactory, final ManagerCache managerCache, final AsyncIndexService indexService, final EntityManager em, final UUID applicationId, final EntityRef headEntity) { ++ public CpRelationManager( final MetricsFactory metricsFactory, final ManagerCache managerCache, final ++ EntityCollectionManagerFactory entityCollectionManagerFactory, final GraphManagerFactory graphManagerFactory, final AsyncIndexService indexService, final EntityManager em, final UUID applicationId, final EntityRef headEntity) { - public CpRelationManager init( - EntityManager em, - UUID applicationId, - EntityRef headEntity, - IndexBucketLocator indexBucketLocator, - MetricsFactory metricsFactory, - ManagerCache managerCache, - EntityCollectionManagerFactory entityCollectionManagerFactory, - GraphManagerFactory graphManagerFactory ) { Assert.notNull( em, "Entity manager cannot be null" ); Assert.notNull( applicationId, "Application Id cannot be null" ); @@@ -225,30 -141,13 +148,16 @@@ this.managerCache = managerCache; this.applicationScope = CpNamingUtils.getApplicationScope( applicationId ); - this.cass = em.getCass(); // TODO: eliminate need for this via Core Persistence - this.indexBucketLocator = indexBucketLocator; // TODO: this also - this.metricsFactory = metricsFactory; - this.updateCollectionTimer = metricsFactory - .getTimer( CpRelationManager.class, "relation.manager.es.update.collection" ); - this.createConnectionTimer = metricsFactory - .getTimer( CpRelationManager.class, "relation.manager.es.create.connection.timer" ); - this.cassConnectionDelete = metricsFactory - .getTimer( CpRelationManager.class, "relation.manager.cassandra.delete.connection.batch.timer" ); - this.esDeleteConnectionTimer = metricsFactory.getTimer(CpRelationManager.class, "relation.manager.es.delete.connection.batch.timer" ); - // load the Core Persistence version of the head entity as well - this.headEntityScope = getCollectionScopeNameFromEntityType( - applicationScope.getApplication(), headEntity.getType()); + this.entityCollectionManagerFactory = entityCollectionManagerFactory; + this.graphManagerFactory = graphManagerFactory; + + this.metricsFactory = metricsFactory; + this.updateCollectionTimer = + metricsFactory.getTimer( CpRelationManager.class, "relation.manager.es.update.collection" ); + if ( logger.isDebugEnabled() ) { - logger.debug( "Loading head entity {}:{} from scope\n app {}\n owner {}\n name {}", - new Object[] { - headEntity.getType(), - headEntity.getUuid(), - headEntityScope.getApplication(), - headEntityScope.getOwner(), - headEntityScope.getName() + logger.debug( "Loading head entity {}:{} from app {}", new Object[] { + headEntity.getType(), headEntity.getUuid(), applicationScope } ); } @@@ -339,8 -227,7 +237,7 @@@ public Observable<Edge> call( final String edgeType ) { return gm.loadEdgesToTarget( new SimpleSearchByEdgeType( cpHeadEntity.getId(), edgeType, Long.MAX_VALUE, - SearchByEdgeType.Order.DESCENDING, null ) ); + SearchByEdgeType.Order.DESCENDING, Optional.<Edge>absent() ) ); - } } ); @@@ -538,24 -295,27 +305,6 @@@ } - private boolean moreThanOneOutboundConnection( EntityRef source, String connectionType ) { - private boolean moreThanOneInboundConnection( EntityRef target, String connectionType ) { -- - Id sourceId = new SimpleId( source.getUuid(), source.getType() ); - Id targetId = new SimpleId( target.getUuid(), target.getType() ); -- -- GraphManager gm = managerCache.getGraphManager( applicationScope ); - - Observable<Edge> edgesFromSource = gm.loadEdgesFromSource( new SimpleSearchByEdgeType( - sourceId, - CpNamingUtils.getEdgeTypeFromConnectionType( connectionType ), - System.currentTimeMillis(), - SearchByEdgeType.Order.DESCENDING, - Optional.<Edge>absent() ) ); // last -- - int count = edgesFromSource.take( 2 ).count().toBlocking().last(); - Observable<Edge> edgesToTarget = gm.loadEdgesToTarget( - new SimpleSearchByEdgeType( targetId, CpNamingUtils.getEdgeTypeFromConnectionType( connectionType ), - System.currentTimeMillis(), SearchByEdgeType.Order.DESCENDING, null ) ); // last -- - return count > 1; - Iterator<Edge> iterator = edgesToTarget.toBlocking().getIterator(); - int count = 0; - while ( iterator.hasNext() ) { - iterator.next(); - if ( count++ > 1 ) { - return true; - } - } - return false; -- } -- @Override public Set<String> getCollections() throws Exception { @@@ -927,43 -637,29 +626,31 @@@ headEntity = em.validate( headEntity ); - CollectionInfo collection = - getDefaultSchema().getCollection( headEntity.getType(), collName ); + CollectionInfo collection = getDefaultSchema().getCollection( headEntity.getType(), collName ); if ( collection == null ) { - throw new RuntimeException( "Cannot find collection-info for '" + collName - + "' of " + headEntity.getType() + ":" + headEntity .getUuid() ); + throw new RuntimeException( + "Cannot find collection-info for '" + collName + "' of " + headEntity.getType() + ":" + headEntity + .getUuid() ); } - query.setEntityType( collection.getType() ); - query = adjustQuery( query ); - /** - * It's a graph query, execute the graph query executor. - * - * TODO refactor all of this away into commands - */ - if(query.isGraphQuery()){ - final QueryExecutor executor = new CollectionGraphQueryExecutor( this.entityCollectionManagerFactory,graphManagerFactory, applicationScope, headEntity, query.getCursor(), collName, query.getLimit() ); - return executor.next(); - } + final SearchEdge searchEdge = createCollectionSearchEdge( cpHeadEntity.getId(), collName ); + final ApplicationEntityIndex ei = managerCache.getEntityIndex( applicationScope ); + final SearchTypes types = SearchTypes.fromTypes( collection.getType() ); - final IndexScope indexScope = new IndexScopeImpl( - cpHeadEntity.getId(), - CpNamingUtils.getCollectionScopeNameFromCollectionName( collName ) ); - - final EntityIndex ei = managerCache.getEntityIndex( applicationScope ); + logger.debug( "Searching scope {}", searchEdge ); - final SearchTypes types = SearchTypes.fromTypes( collection.getType() ); + query.setEntityType( collection.getType() ); + query = adjustQuery( query ); - logger.debug( "Searching scope {}:{}", - indexScope.getOwner().toString(), indexScope.getName() ); + + - final CollectionResultsLoaderFactoryImpl resultsLoaderFactory = new CollectionResultsLoaderFactoryImpl( managerCache ); + final CollectionResultsLoaderFactoryImpl resultsLoaderFactory = + new CollectionResultsLoaderFactoryImpl( managerCache ); //execute the query and return our next result http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ec3a710a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpWalker.java ---------------------------------------------------------------------- diff --cc stack/core/src/main/java/org/apache/usergrid/corepersistence/CpWalker.java index b2354a6,3f2c9d6..15e4341 --- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpWalker.java +++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpWalker.java @@@ -103,56 -103,39 +105,39 @@@ public class CpWalker edgeType = CpNamingUtils.getEdgeTypeFromCollectionName( collectionName ); } - Observable<String> edgeTypes = gm.getEdgeTypesFromSource( - new SimpleSearchEdgeType( applicationId, edgeType, null ) ); - - edgeTypes.flatMap( new Func1<String, Observable<Edge>>() { - @Override - public Observable<Edge> call( final String edgeType ) { - - logger.debug( "Loading edges of type {} from node {}", edgeType, applicationId ); - - return gm.loadEdgesFromSource( new SimpleSearchByEdgeType( - applicationId, edgeType, Long.MAX_VALUE, order , Optional.<Edge>absent() ) ); - - } - - } ).parallel( new Func1<Observable<Edge>, Observable<Edge>>() { - - @Override - public Observable<Edge> call( final Observable<Edge> edgeObservable ) { // process edges in parallel - return edgeObservable.doOnNext( new Action1<Edge>() { // visit and update then entity - - @Override - public void call( Edge edge ) { - - logger.info( "Re-indexing edge {}", edge ); - - EntityRef targetNodeEntityRef = new SimpleEntityRef( - edge.getTargetNode().getType(), - edge.getTargetNode().getUuid() ); - - Entity entity; - try { - entity = em.get( targetNodeEntityRef ); - } - catch ( Exception ex ) { - logger.error( "Error getting sourceEntity {}:{}, continuing", - targetNodeEntityRef.getType(), - targetNodeEntityRef.getUuid() ); - return; - } - if(entity == null){ - return; - } - String collName = CpNamingUtils.getCollectionName( edge.getType() ); - visitor.visitCollectionEntry( em, collName, entity ); - } - } ); - } - }, Schedulers.io() ) + Observable<Edge> edges = gm.getEdgeTypesFromSource( + new SimpleSearchEdgeType( applicationId, edgeType, null ) ).flatMap( emittedEdgeType -> { + + logger.debug( "Loading edges of type {} from node {}", edgeType, applicationId ); + + return gm.loadEdgesFromSource( - new SimpleSearchByEdgeType( applicationId, emittedEdgeType, Long.MAX_VALUE, order, null ) ); ++ new SimpleSearchByEdgeType( applicationId, emittedEdgeType, Long.MAX_VALUE, order, Optional.absent() ) ); + } ).flatMap( edge -> { + //run each edge through it's own scheduler, up to 100 at a time + return Observable.just( edge ).doOnNext( edgeValue -> { + logger.info( "Re-indexing edge {}", edgeValue ); + + EntityRef targetNodeEntityRef = + new SimpleEntityRef( edgeValue.getTargetNode().getType(), edgeValue.getTargetNode().getUuid() ); + + Entity entity; + try { + entity = em.get( targetNodeEntityRef ); + } + catch ( Exception ex ) { + logger.error( "Error getting sourceEntity {}:{}, continuing", targetNodeEntityRef.getType(), + targetNodeEntityRef.getUuid() ); + return; + } + if ( entity == null ) { + return; + } + String collName = getNameFromEdgeType( edgeValue.getType() ); + visitor.visitCollectionEntry( em, collName, entity ); + } ).subscribeOn( Schedulers.io() ); + }, 100 ); // wait for it to complete - .toBlocking().lastOrDefault( null ); // end foreach on edges + edges.toBlocking().lastOrDefault( null ); // end foreach on edges } } http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ec3a710a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/CollectionResultsLoaderFactoryImpl.java ----------------------------------------------------------------------
