merge from 405
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/75d61b27 Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/75d61b27 Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/75d61b27 Branch: refs/heads/USERGRID-480 Commit: 75d61b270efca8cd7182543a977deafcd136582c Parents: f7e78f4 dcf4693 Author: Shawn Feldman <[email protected]> Authored: Fri Mar 20 08:44:34 2015 -0600 Committer: Shawn Feldman <[email protected]> Committed: Fri Mar 20 08:44:34 2015 -0600 ---------------------------------------------------------------------- stack/core/pom.xml | 26 +- .../corepersistence/CpEntityManager.java | 2 +- .../corepersistence/CpEntityManagerFactory.java | 7 +- .../corepersistence/CpRelationManager.java | 24 +- .../usergrid/corepersistence/CpWalker.java | 81 ++--- .../events/EntityVersionDeletedHandler.java | 71 ++-- .../migration/EntityTypeMappingMigration.java | 41 +-- .../persistence/ObservableIterator.java | 1 + .../migration/EntityTypeMappingMigrationIT.java | 2 +- .../PerformanceEntityRebuildIndexTest.java | 4 +- stack/corepersistence/collection/pom.xml | 14 +- .../impl/EntityCollectionManagerImpl.java | 10 +- .../collection/impl/EntityDeletedTask.java | 20 +- .../impl/EntityVersionCleanupTask.java | 40 +-- .../impl/EntityVersionCreatedTask.java | 26 +- .../mvcc/stage/write/WriteCommit.java | 2 +- .../mvcc/stage/write/WriteUniqueVerify.java | 8 +- .../MvccEntitySerializationStrategyImpl.java | 92 ++--- .../MvccEntitySerializationStrategyV3Impl.java | 94 ++--- .../UniqueValueSerializationStrategyImpl.java | 8 - .../migration/MvccEntityDataMigrationImpl.java | 171 ++++------ .../collection/util/EntityUtils.java | 72 ---- .../mvcc/stage/AbstractEntityStageTest.java | 2 +- .../mvcc/stage/AbstractMvccEntityStageTest.java | 2 +- .../mvcc/stage/TestEntityGenerator.java | 2 +- .../persistence/collection/rx/ParallelTest.java | 10 +- ...MvccEntitySerializationStrategyImplTest.java | 4 +- ...ccEntitySerializationStrategyV1ImplTest.java | 4 +- ...ccEntitySerializationStrategyV2ImplTest.java | 2 +- .../impl/SerializationComparison.java | 4 +- ...ctMvccEntityDataMigrationV1ToV3ImplTest.java | 2 +- .../collection/util/InvalidEntityGenerator.java | 1 + stack/corepersistence/common/pom.xml | 15 +- .../astyanax/MultiKeyColumnNameIterator.java | 4 +- .../MultiKeyColumnNameIteratorTest.java | 187 +++++----- .../astyanax/MultiRowColumnIteratorTest.java | 50 ++- .../graph/impl/GraphManagerImpl.java | 6 +- .../graph/impl/stage/EdgeMetaRepairImpl.java | 2 + .../impl/stage/NodeDeleteListenerImpl.java | 2 +- .../impl/migration/EdgeDataMigrationImpl.java | 87 +++-- .../persistence/graph/GraphManagerIT.java | 17 +- .../graph/GraphManagerShardConsistencyIT.java | 2 +- .../usergrid/persistence/graph/SimpleTest.java | 12 +- .../migration/EdgeDataMigrationImplTest.java | 2 +- stack/corepersistence/model/pom.xml | 1 - .../persistence/model/util/EntityUtils.java | 72 ++++ stack/corepersistence/pom.xml | 8 +- stack/corepersistence/queryindex/pom.xml | 6 - .../persistence/index/query/EntityResults.java | 108 ------ .../persistence/index/query/Results.java | 148 -------- .../persistence/index/utils/ListUtils.java | 6 +- .../persistence/index/guice/IndexTestFig.java | 57 ++++ .../index/guice/TestIndexModule.java | 8 +- .../index/impl/CorePerformanceIT.java | 339 ------------------- .../impl/EntityConnectionIndexImplTest.java | 302 ----------------- .../persistence/index/impl/EntityIndexTest.java | 32 +- .../index/impl/IndexLoadTestsIT.java | 138 ++++++++ stack/pom.xml | 8 +- .../management/importer/ImportServiceImpl.java | 34 +- .../impl/ApplicationQueueManagerImpl.java | 195 +++++------ .../setup/ConcurrentProcessSingleton.java | 16 +- 61 files changed, 899 insertions(+), 1814 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/75d61b27/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/75d61b27/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/75d61b27/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java ---------------------------------------------------------------------- diff --cc stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java index df01c47,c4e970d..f505fa3 --- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java +++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java @@@ -1055,9 -1058,9 +1055,9 @@@ public class CpRelationManager implemen cpHeadEntity.getId(), edgeType, targetEntity.getId(), System.currentTimeMillis() ); GraphManager gm = managerCache.getGraphManager( applicationScope ); - gm.writeEdge( edge ).toBlockingObservable().last(); + gm.writeEdge( edge ).toBlocking().last(); - EntityIndex ei = managerCache.getEntityIndex( applicationScope ); + ApplicationEntityIndex ei = managerCache.getEntityIndex( applicationScope ); EntityIndexBatch batch = ei.createBatch(); // Index the new connection in app|source|type context @@@ -1287,9 -1290,9 +1287,9 @@@ System.currentTimeMillis() ); GraphManager gm = managerCache.getGraphManager( applicationScope ); - gm.deleteEdge( edge ).toBlockingObservable().last(); + gm.deleteEdge( edge ).toBlocking().last(); - final EntityIndex ei = managerCache.getEntityIndex( applicationScope ); + final ApplicationEntityIndex ei = managerCache.getEntityIndex( applicationScope ); final EntityIndexBatch batch = ei.createBatch(); // Deindex the connection in app|source|type context http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/75d61b27/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionDeletedHandler.java ---------------------------------------------------------------------- diff --cc stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionDeletedHandler.java index ece7562,23f5a32..f0dd136 --- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionDeletedHandler.java +++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionDeletedHandler.java @@@ -17,21 -17,18 +17,20 @@@ */ package org.apache.usergrid.corepersistence.events; - import com.google.inject.Inject; - import com.google.inject.Singleton; import java.util.List; + + import org.slf4j.Logger; + import org.slf4j.LoggerFactory; + import org.apache.usergrid.corepersistence.CpEntityManagerFactory; - import static org.apache.usergrid.corepersistence.CoreModule.EVENTS_DISABLED; import org.apache.usergrid.persistence.EntityManagerFactory; import org.apache.usergrid.persistence.collection.CollectionScope; - import org.apache.usergrid.persistence.collection.MvccEntity; import org.apache.usergrid.persistence.collection.MvccLogEntry; import org.apache.usergrid.persistence.collection.event.EntityVersionDeleted; +import org.apache.usergrid.persistence.collection.serialization.SerializationFig; +import org.apache.usergrid.persistence.index.ApplicationEntityIndex; import org.apache.usergrid.persistence.index.EntityIndex; - import org.apache.usergrid.persistence.index.EntityIndexBatch; import org.apache.usergrid.persistence.index.IndexScope; import org.apache.usergrid.persistence.index.impl.IndexScopeImpl; import org.apache.usergrid.persistence.model.entity.Id; @@@ -84,28 -78,21 +80,21 @@@ public class EntityVersionDeletedHandle } ); } - CpEntityManagerFactory cpemf = (CpEntityManagerFactory)emf; + CpEntityManagerFactory cpemf = ( CpEntityManagerFactory ) emf; - final EntityIndex ei = cpemf.getManagerCache().getEntityIndex( scope ); + final ApplicationEntityIndex ei = cpemf.getManagerCache().getEntityIndex( scope ); - final IndexScope indexScope = new IndexScopeImpl( - new SimpleId(scope.getOwner().getUuid(), scope.getOwner().getType()), - scope.getName() - ); - - Observable.from( entityVersions ) - .collect( ei.createBatch(), new Action2<EntityIndexBatch, MvccLogEntry>() { - @Override - public void call( final EntityIndexBatch entityIndexBatch, final MvccLogEntry mvccLogEntry ) { - entityIndexBatch.deindex( indexScope, mvccLogEntry.getEntityId(), mvccLogEntry.getVersion() ); - } - } ).doOnNext( new Action1<EntityIndexBatch>() { - @Override - public void call( final EntityIndexBatch entityIndexBatch ) { + final IndexScope indexScope = + new IndexScopeImpl( new SimpleId( scope.getOwner().getUuid(), scope.getOwner().getType() ), + scope.getName() ); + + //create our batch, and then collect all of them into a single batch + Observable.from( entityVersions ).collect( () -> ei.createBatch(), ( entityIndexBatch, mvccLogEntry ) -> { + entityIndexBatch.deindex( indexScope, mvccLogEntry.getEntityId(), mvccLogEntry.getVersion() ); + } ) + //after our batch is collected, execute it + .doOnNext( entityIndexBatch -> { entityIndexBatch.execute(); - } - } ).toBlocking().last(); + } ).toBlocking().last(); } - - } http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/75d61b27/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRebuildIndexTest.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/75d61b27/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/CorePerformanceIT.java ---------------------------------------------------------------------- diff --cc stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/CorePerformanceIT.java index 303d481,0000000..e69de29 mode 100644,000000..100644 --- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/CorePerformanceIT.java +++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/CorePerformanceIT.java http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/75d61b27/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityConnectionIndexImplTest.java ---------------------------------------------------------------------- diff --cc stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityConnectionIndexImplTest.java index 8e8d6c0,0000000..e69de29 mode 100644,000000..100644 --- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityConnectionIndexImplTest.java +++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityConnectionIndexImplTest.java http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/75d61b27/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java ----------------------------------------------------------------------
