Merge remote-tracking branch 'origin/two-dot-o-dev' into USERGRID-509
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/86dff76c Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/86dff76c Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/86dff76c Branch: refs/heads/two-dot-o-dev Commit: 86dff76c85c5c5cfb6f8badc3d431609113e0c8d Parents: 535003c c1643a7 Author: Todd Nine <[email protected]> Authored: Tue Mar 31 16:57:15 2015 -0600 Committer: Todd Nine <[email protected]> Committed: Tue Mar 31 16:57:15 2015 -0600 ---------------------------------------------------------------------- .../main/resources/usergrid-default.properties | 2 + .../batch/service/JobSchedulerService.java | 28 +- .../corepersistence/ApplicationIdCache.java | 52 +++ .../corepersistence/ApplicationIdCacheImpl.java | 135 +++++++ .../usergrid/corepersistence/CoreModule.java | 9 +- .../corepersistence/CpEntityManager.java | 4 + .../corepersistence/CpEntityManagerFactory.java | 359 +++++++++--------- .../usergrid/corepersistence/CpSetup.java | 16 +- .../corepersistence/OrgApplicationCache.java | 67 ---- .../OrgApplicationCacheImpl.java | 181 --------- .../migration/AppInfoMigrationPlugin.java | 343 +++++++++++++++++ .../migration/AppInfoVersions.java | 40 ++ .../migration/CoreDataVersions.java | 8 +- .../migration/MigrationModuleVersionPlugin.java | 22 +- .../rx/impl/AllApplicationsObservableImpl.java | 25 +- .../corepersistence/util/CpNamingUtils.java | 34 +- .../usergrid/exception/ConflictException.java | 36 ++ .../usergrid/persistence/EntityManager.java | 1 - .../persistence/EntityManagerFactory.java | 31 +- .../org/apache/usergrid/persistence/Schema.java | 5 +- .../cassandra/EntityManagerFactoryImpl.java | 25 +- .../cassandra/EntityManagerImpl.java | 1 - .../persistence/cassandra/SetupImpl.java | 2 - .../main/resources/usergrid-core-context.xml | 1 + .../java/org/apache/usergrid/CoreITSetup.java | 2 - .../org/apache/usergrid/CoreITSetupImpl.java | 37 +- .../rx/AllEntitiesInSystemObservableIT.java | 44 +-- .../rx/ApplicationObservableTestIT.java | 2 - .../apache/usergrid/persistence/CounterIT.java | 9 +- .../usergrid/persistence/EntityManagerIT.java | 10 +- .../PerformanceEntityRebuildIndexTest.java | 25 +- .../cassandra/EntityManagerFactoryImplIT.java | 86 +++-- .../persistence/map/guice/MapModule.java | 4 +- .../map/impl/MapManagerFactoryImpl.java | 29 +- .../usergrid/persistence/index/EntityIndex.java | 2 + .../impl/EsApplicationEntityIndexImpl.java | 2 +- .../index/impl/EsEntityIndexBatchImpl.java | 1 + .../index/impl/EsEntityIndexImpl.java | 77 ++-- .../index/impl/EsIndexBufferConsumerImpl.java | 37 +- .../index/migration/EsIndexMigrationPlugin.java | 2 +- .../usergrid/mongo/protocol/OpDelete.java | 4 +- .../apache/usergrid/mongo/protocol/OpQuery.java | 6 +- .../usergrid/mongo/protocol/OpUpdate.java | 4 +- stack/pom.xml | 1 + .../apache/usergrid/rest/MigrateResource.java | 56 +-- .../rest/applications/ApplicationResource.java | 38 +- .../rest/applications/ServiceResource.java | 8 +- .../exceptions/ConflictExceptionMapper.java | 35 ++ .../exceptions/NotFoundExceptionMapper.java | 35 ++ .../applications/ApplicationResource.java | 15 - .../applications/ApplicationsResource.java | 34 +- .../imports/FileIncludesResource.java | 9 - .../applications/imports/ImportsResource.java | 79 ++-- .../organizations/OrganizationResource.java | 23 +- .../OAuth2AccessTokenSecurityFilter.java | 2 +- .../apache/usergrid/rest/IndexResourceIT.java | 1 + .../applications/ApplicationDeleteTest.java | 373 ++++++++++++++++++- .../applications/ApplicationResourceIT.java | 63 ++-- .../collection/users/PermissionsResourceIT.java | 15 - .../collection/users/UserResourceIT.java | 12 +- .../rest/applications/queries/OrderByTest.java | 2 +- .../rest/management/ImportResourceIT.java | 151 ++++---- .../endpoints/ApplicationsResource.java | 34 +- .../endpoints/OrganizationResource.java | 6 +- .../endpoints/mgmt/ApplicationResource.java | 50 ++- .../endpoints/mgmt/ApplicationsResource.java | 58 +++ .../endpoints/mgmt/ManagementResponse.java | 72 ++++ .../mgmt/OrganizationApplicationResponse.java | 60 +++ .../endpoints/mgmt/OrganizationResource.java | 5 +- stack/rest/src/test/resources/log4j.properties | 10 +- .../resources/testImportCorrect.testCol.1.json | 18 - ...testImportInvalidJson.testApplication.3.json | 153 -------- .../testimport-bad-json-testapp.3.json | 153 ++++++++ .../src/test/resources/testimport-bad-json.json | 67 ++++ .../resources/testimport-correct-testcol.1.json | 18 + .../test/resources/testimport-test-app.2.json | 56 +++ .../usergrid/management/ApplicationInfo.java | 4 + .../usergrid/management/ManagementService.java | 10 +- .../cassandra/ManagementServiceImpl.java | 228 +++++++----- .../management/importer/FileImportJob.java | 6 +- .../management/importer/FileImportTracker.java | 6 +- .../usergrid/management/importer/ImportJob.java | 4 +- .../management/importer/ImportServiceImpl.java | 44 +-- .../usergrid/security/AuthPrincipalType.java | 3 +- .../shiro/PrincipalCredentialsToken.java | 20 +- .../services/ServiceManagerFactory.java | 4 - .../org/apache/usergrid/ServiceITSetup.java | 3 + .../org/apache/usergrid/ServiceITSetupImpl.java | 34 +- .../migration/AppInfoMigrationPluginTest.java | 218 +++++++++++ .../apache/usergrid/management/EmailFlowIT.java | 2 - .../cassandra/ApplicationCreatorIT.java | 4 +- .../cassandra/ManagementServiceIT.java | 13 +- .../management/export/ExportServiceIT.java | 8 +- .../importer/FileImportTrackerTest.java | 12 +- .../management/importer/ImportCollectionIT.java | 1 - .../importer/ImportConnectionsTest.java | 7 +- .../security/tokens/TokenServiceIT.java | 2 - .../usergrid/services/ServiceFactoryIT.java | 8 +- .../usergrid/services/ServiceRequestIT.java | 9 +- .../src/test/resources/log4j.properties | 6 +- .../setup/ConcurrentProcessSingleton.java | 39 +- .../org/apache/usergrid/tools/AppAudit.java | 3 +- .../org/apache/usergrid/tools/CredExport.java | 2 +- .../org/apache/usergrid/tools/DupOrgRepair.java | 1 - .../java/org/apache/usergrid/tools/Import.java | 14 +- .../usergrid/tools/UniqueIndexCleanup.java | 6 +- 106 files changed, 2875 insertions(+), 1368 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/86dff76c/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/86dff76c/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java ---------------------------------------------------------------------- diff --cc stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java index 318c859,c613e44..15fb386 --- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java +++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java @@@ -210,8 -204,8 +201,8 @@@ public class CpEntityManagerFactory imp } @Override - public UUID createApplication(String organizationName, String name) throws Exception { - return createApplication( organizationName, name, null ); + public Entity createApplicationV2(String organizationName, String name) throws Exception { - return createApplicationV2(organizationName, name, null); ++ return createApplicationV2( organizationName, name, null ); } @@@ -231,10 -225,9 +222,9 @@@ UUID applicationId = UUIDGenerator.newTimeUUID(); logger.debug( "New application orgName {} orgAppName {} id {} ", -- new Object[] { orgName, name, applicationId.toString() } ); ++ new Object[] { orgName, name, applicationId.toString() } ); - initializeApplication( orgName, applicationId, appName, properties ); - return applicationId; + return initializeApplicationV2(orgName, applicationId, appName, properties); } @@@ -425,23 -425,29 +422,24 @@@ public Map<String, UUID> getApplications(boolean deleted) throws Exception { - Map<String, UUID> appMap = new HashMap<String, UUID>(); + Map<String, UUID> appMap = new HashMap<>(); - ApplicationScope mgmtAppScope = CpNamingUtils.getApplicationScope(getManagementAppId()); - GraphManager gm = managerCache.getGraphManager(mgmtAppScope); + ApplicationScope appScope = CpNamingUtils.getApplicationScope( CpNamingUtils.SYSTEM_APP_ID ); + GraphManager gm = managerCache.getGraphManager( appScope ); - EntityManager em = getEntityManager(getManagementAppId()); - Application mgmtApp = em.getApplication(); - Id fromEntityId = new SimpleId( mgmtApp.getUuid(), mgmtApp.getType() ); + EntityManager em = getEntityManager( CpNamingUtils.SYSTEM_APP_ID ); + Application app = em.getApplication(); + Id fromEntityId = new SimpleId( app.getUuid(), app.getType() ); - final String scopeName; +// final String scopeName; final String edgeType; + if ( deleted ) { - edgeType = CpNamingUtils.getEdgeTypeFromCollectionName( - CpNamingUtils.DELETED_APPLICATION_INFOS); - scopeName = CpNamingUtils.getCollectionScopeNameFromCollectionName( - CpNamingUtils.DELETED_APPLICATION_INFOS); - + edgeType = CpNamingUtils.getEdgeTypeFromCollectionName( CpNamingUtils.DELETED_APPINFOS ); +// scopeName = CpNamingUtils.getCollectionScopeNameFromCollectionName(CpNamingUtils.DELETED_APPINFOS); } else { - edgeType = CpNamingUtils.getEdgeTypeFromCollectionName( - CpNamingUtils.APPLICATION_INFOS ); - scopeName = CpNamingUtils.getCollectionScopeNameFromCollectionName( - CpNamingUtils.APPLICATION_INFOS); + edgeType = CpNamingUtils.getEdgeTypeFromCollectionName( CpNamingUtils.APPINFOS ); +// scopeName = CpNamingUtils.getCollectionScopeNameFromCollectionName(CpNamingUtils.APPINFOS); } logger.debug("getApplications(): Loading edges of edgeType {} from {}:{}", @@@ -463,14 -471,17 +463,14 @@@ edge.getTargetNode().getType(), edge.getTargetNode().getUuid() }); - CollectionScope collScope = new CollectionScopeImpl( - mgmtAppScope.getApplication(), - mgmtAppScope.getApplication(), - scopeName); + - org.apache.usergrid.persistence.model.entity.Entity e = + org.apache.usergrid.persistence.model.entity.Entity appInfo = - managerCache.getEntityCollectionManager( collScope ).load( targetId ) + managerCache.getEntityCollectionManager( appScope ).load( targetId ) .toBlocking().lastOrDefault(null); - if ( e == null ) { - logger.warn("Applicaion {} in index but not found in collections", targetId ); + if ( appInfo == null ) { + logger.warn("Application {} in index but not found in collections", targetId ); continue; } @@@ -520,8 -532,8 +521,8 @@@ @Override public boolean updateServiceProperties(Map<String, String> properties) { - EntityManager em = getEntityManager( CpNamingUtils.SYSTEM_APP_ID); - Query q = Query.fromQL("select *"); + EntityManager em = getEntityManager(getManagementAppId()); - Query q = Query.fromQL("select *"); ++ Query q = Query.fromQL( "select *"); Results results = null; try { results = em.searchCollection( em.getApplicationRef(), "propertymaps", q); @@@ -569,10 -581,10 +570,10 @@@ @Override public boolean deleteServiceProperty(String name) { - EntityManager em = getEntityManager( CpNamingUtils.SYSTEM_APP_ID); + EntityManager em = getEntityManager(getManagementAppId()); -- Query q = Query.fromQL("select *"); ++ Query q = Query.fromQL( "select *"); Results results = null; try { results = em.searchCollection( em.getApplicationRef(), "propertymaps", q); @@@ -751,7 -754,7 +743,7 @@@ @Override public void addIndex(final UUID applicationId,final String indexSuffix,final int shards,final int replicas, final String writeConsistency){ -- entityIndex.addIndex(indexSuffix, shards, replicas,writeConsistency); ++ entityIndex.addIndex( indexSuffix, shards, replicas, writeConsistency); } @Override @@@ -759,16 -762,32 +751,33 @@@ // could use any collection scope here, does not matter EntityCollectionManager ecm = getManagerCache().getEntityCollectionManager( - new ApplicationScopeImpl( - new SimpleId( CpNamingUtils.SYSTEM_APP_ID, "application")) - ); - new CollectionScopeImpl( - new SimpleId( getManagementAppId(), "application"), - new SimpleId( getManagementAppId(), "application"), - "dummy" - )); ++ new ApplicationScopeImpl( new SimpleId( CpNamingUtils.SYSTEM_APP_ID, "application" ) ) ); return ecm.getHealth(); } @Override + public UUID createApplication(String organizationName, String name) throws Exception { + throw new UnsupportedOperationException("Not supported in v2"); + } + + + @Override + public UUID createApplication( + String organizationName, String name, Map<String, Object> properties) throws Exception { + throw new UnsupportedOperationException("Not supported in v2"); + } + + @Override + public UUID initializeApplication( + String orgName, UUID appId, String appName, Map<String, Object> props) throws Exception { + throw new UnsupportedOperationException("Not supported in v2"); + } + ++ ++ @Override + public Health getIndexHealth() { + return entityIndex.getIndexHealth(); + } } http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/86dff76c/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllApplicationsObservableImpl.java ---------------------------------------------------------------------- diff --cc stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllApplicationsObservableImpl.java index 9c6aee7,d872b71..5151857 --- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllApplicationsObservableImpl.java +++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllApplicationsObservableImpl.java @@@ -69,25 -73,25 +71,21 @@@ public class AllApplicationsObservableI @Override public Observable<ApplicationScope> getData() { - //emit our 3 hard coded applications that are used the manage the system first. + //emit our hard coded applications that are used the manage the system first. //this way consumers can perform whatever work they need to on the root system first - final Observable<ApplicationScope> systemIds = Observable.from( Arrays - .asList( getApplicationScope( CpNamingUtils.DEFAULT_APPLICATION_ID ), + final Observable<ApplicationScope> systemIds = Observable.from( + Arrays.asList( getApplicationScope( CpNamingUtils.MANAGEMENT_APPLICATION_ID ), - getApplicationScope( CpNamingUtils.SYSTEM_APP_ID ) ) ); - - - final ApplicationScope appScope = getApplicationScope( CpNamingUtils.SYSTEM_APP_ID ); + getApplicationScope( CpNamingUtils.SYSTEM_APP_ID ))); // still need deprecated system app here + final ApplicationScope appScope = getApplicationScope( CpNamingUtils.MANAGEMENT_APPLICATION_ID ); - final CollectionScope appInfoCollectionScope = - new CollectionScopeImpl( appScope.getApplication(), appScope.getApplication(), - CpNamingUtils.getCollectionScopeNameFromCollectionName( CpNamingUtils.APPLICATION_INFOS) ); - final EntityCollectionManager collectionManager = - entityCollectionManagerFactory.createCollectionManager( appInfoCollectionScope ); + entityCollectionManagerFactory.createCollectionManager( appScope ); - final GraphManager gm = graphManagerFactory.createEdgeManager(appScope); - - String edgeType = CpNamingUtils.getEdgeTypeFromCollectionName( CpNamingUtils.APPINFOS ); + String edgeType = CpNamingUtils.getEdgeTypeFromCollectionName( CpNamingUtils.APPLICATION_INFOS ); Id rootAppId = appScope.getApplication(); http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/86dff76c/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/CpNamingUtils.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/86dff76c/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/86dff76c/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java ---------------------------------------------------------------------- diff --cc stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java index 95e7e34,d571359..4182732 --- a/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java +++ b/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java @@@ -21,8 -21,8 +21,6 @@@ import java.util.Map import java.util.UUID; import org.apache.usergrid.persistence.core.util.Health; --import org.apache.usergrid.persistence.index.EntityIndex; --import org.springframework.context.ApplicationContext; /** @@@ -147,13 -166,9 +164,11 @@@ public interface EntityManagerFactory public Health getEntityStoreHealth(); + public Health getIndexHealth(); + - void restoreApplication(UUID applicationId) throws Exception; - public interface ProgressObserver { - public void onProgress( EntityRef entity); + public void onProgress( EntityRef entity); } } http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/86dff76c/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImpl.java ---------------------------------------------------------------------- diff --cc stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImpl.java index 6627fa4,e8aae01..882e7d9 --- a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImpl.java +++ b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImpl.java @@@ -444,15 -445,8 +445,15 @@@ public class EntityManagerFactoryImpl i throw new UnsupportedOperationException("Not supported in v1."); } + + @Override + public Health getIndexHealth() { + return null; + } + + @Override - public void restoreApplication(UUID applicationId) throws Exception { + public Entity restoreApplication(UUID applicationId) throws Exception { throw new UnsupportedOperationException("Not supported in v1"); } http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/86dff76c/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/86dff76c/stack/core/src/test/java/org/apache/usergrid/corepersistence/rx/AllEntitiesInSystemObservableIT.java ---------------------------------------------------------------------- diff --cc stack/core/src/test/java/org/apache/usergrid/corepersistence/rx/AllEntitiesInSystemObservableIT.java index 3e6a77f,b2b8aaf..e3c8e1c --- a/stack/core/src/test/java/org/apache/usergrid/corepersistence/rx/AllEntitiesInSystemObservableIT.java +++ b/stack/core/src/test/java/org/apache/usergrid/corepersistence/rx/AllEntitiesInSystemObservableIT.java @@@ -99,17 -100,18 +100,18 @@@ public class AllEntitiesInSystemObserva @Override public void call( final EntityIdScope entityIdScope ) { assertNotNull(entityIdScope); - assertNotNull(entityIdScope.getCollectionScope()); + assertNotNull(entityIdScope.getApplicationScope()); assertNotNull(entityIdScope.getId()); - //we should only emit each node once - if ( entityIdScope.getId().getType().equals( type1 ) ) { - assertTrue( "Element should be present on removal", type1Identities.remove(entityIdScope.getId() ) ); - } - else if ( entityIdScope.getId().getType().equals( type2 ) ) { - assertTrue( "Element should be present on removal", type2Identities.remove(entityIdScope.getId() ) ); - } - + // we should only emit each node once + if ( entityIdScope.getId().getType().equals( type1 ) ) { + assertTrue( "Element should be present on removal", + type1Identities.remove(entityIdScope.getId() ) ); + } + else if ( entityIdScope.getId().getType().equals( type2 ) ) { + assertTrue( "Element should be present on removal", + type2Identities.remove(entityIdScope.getId() ) ); + } } } ).toBlocking().lastOrDefault( null ); http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/86dff76c/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java ---------------------------------------------------------------------- diff --cc stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java index 7e8d8f4,b2baac9..0377f40 --- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java +++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java @@@ -206,18 -211,21 +206,16 @@@ public class EsIndexBufferConsumerImpl Timer.Context time = flushTimer.time(); - execute( containerList ); + execute(containerList); time.stop(); - } - }) + } ) //ack after we process - .doOnNext(new Action1<List<IndexOperationMessage>>() { - @Override - public void call(final List<IndexOperationMessage> indexOperationMessages) { - bufferQueue.ack(indexOperationMessages); - //release so we know we've done processing - inFlight.addAndGet(-1 * indexOperationMessages.size()); - } - + .doOnNext( indexOperationMessages -> { + bufferQueue.ack( indexOperationMessages ); + //release so we know we've done processing + inFlight.addAndGet( -1 * indexOperationMessages.size() ); - } ) - - .subscribeOn( Schedulers.newThread() ); + } ).subscribeOn( Schedulers.newThread() ); //start in the background http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/86dff76c/stack/rest/src/main/java/org/apache/usergrid/rest/MigrateResource.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/86dff76c/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/applications/ApplicationResource.java ----------------------------------------------------------------------
