merge 408
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/b5ce0856 Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/b5ce0856 Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/b5ce0856 Branch: refs/heads/USERGRID-347 Commit: b5ce08566ba8442f0e69d16c0e7714103c0e8d14 Parents: b35551a 679812e Author: Shawn Feldman <[email protected]> Authored: Thu Mar 19 10:37:14 2015 -0600 Committer: Shawn Feldman <[email protected]> Committed: Thu Mar 19 10:37:14 2015 -0600 ---------------------------------------------------------------------- .../corepersistence/CpEntityManager.java | 5 +- .../corepersistence/CpEntityManagerFactory.java | 39 +- .../usergrid/corepersistence/ManagerCache.java | 3 +- .../usergrid/persistence/EntityManager.java | 3 +- .../cassandra/EntityManagerImpl.java | 4 +- .../org/apache/usergrid/CoreApplication.java | 7 +- .../org/apache/usergrid/CoreITSetupImpl.java | 1 + .../PerformanceEntityRebuildIndexTest.java | 80 ++-- .../index/ApplicationEntityIndex.java | 42 +++ .../usergrid/persistence/index/EntityIndex.java | 46 +-- .../persistence/index/EntityIndexFactory.java | 2 +- .../persistence/index/IndexIdentifier.java | 25 +- .../usergrid/persistence/index/SearchType.java | 49 +++ .../usergrid/persistence/index/SearchTypes.java | 14 +- .../persistence/index/guice/IndexModule.java | 5 +- .../impl/EsApplicationEntityIndexImpl.java | 305 +++++++++++++++ .../index/impl/EsEntityIndexBatchImpl.java | 45 +-- .../index/impl/EsEntityIndexFactoryImpl.java | 27 +- .../index/impl/EsEntityIndexImpl.java | 375 ++----------------- .../persistence/index/impl/EsQueryVistor.java | 88 +++-- .../persistence/index/impl/IndexingUtils.java | 136 ++++--- .../index/impl/CorePerformanceIT.java | 20 +- .../impl/EntityConnectionIndexImplTest.java | 20 +- .../persistence/index/impl/EntityIndexTest.java | 159 +++----- 24 files changed, 756 insertions(+), 744 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b5ce0856/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java ---------------------------------------------------------------------- diff --cc stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java index f76b9fc,41bcf62..e70e538 --- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java +++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java @@@ -15,24 -15,20 +15,25 @@@ */ package org.apache.usergrid.corepersistence; -import com.google.common.base.Optional; -import com.google.common.cache.CacheBuilder; -import com.google.common.cache.CacheLoader; -import com.google.common.cache.LoadingCache; -import com.google.inject.Guice; -import com.google.inject.Injector; -import static java.lang.String.CASE_INSENSITIVE_ORDER; -import java.util.*; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; +import java.util.UUID; import java.util.concurrent.atomic.AtomicBoolean; ++import org.apache.usergrid.persistence.index.ApplicationEntityIndex; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.BeansException; +import org.springframework.context.ApplicationContext; +import org.springframework.context.ApplicationContextAware; + import org.apache.commons.lang.StringUtils; -import org.apache.usergrid.corepersistence.rx.AllEntitiesInSystemObservable; import org.apache.usergrid.corepersistence.util.CpNamingUtils; import org.apache.usergrid.persistence.AbstractEntity; import org.apache.usergrid.persistence.Entity; @@@ -118,15 -105,16 +119,17 @@@ public class CpEntityManagerFactory imp private CassandraService cassandraService; private CounterUtils counterUtils; private Injector injector; ++ private final EntityIndex entityIndex; private final MetricsFactory metricsFactory; public CpEntityManagerFactory( -- final CassandraService cassandraService, final CounterUtils counterUtils, final Injector injector) { ++ final CassandraService cassandraService, final CounterUtils counterUtils, final Injector injector, final EntityIndex entityIndex) { this.cassandraService = cassandraService; this.counterUtils = counterUtils; this.injector = injector; ++ this.entityIndex = entityIndex; this.managerCache = injector.getInstance( ManagerCache.class ); - this.dataMigrationManager = injector.getInstance( DataMigrationManager.class ); this.metricsFactory = injector.getInstance( MetricsFactory.class ); this.orgApplicationCache = new OrgApplicationCacheImpl( this ); @@@ -311,10 -291,9 +316,9 @@@ properties = new TreeMap<String, Object>( CASE_INSENSITIVE_ORDER ); } properties.put( PROPERTY_NAME, appName ); - EntityManager appEm = getEntityManager( applicationId ); + EntityManager appEm = getEntityManager( applicationId); - appEm.create( applicationId, TYPE_APPLICATION, properties ); - appEm.createIndex(); - appEm.create(applicationId, TYPE_APPLICATION, properties ); ++ appEm.create(applicationId, TYPE_APPLICATION, properties); appEm.resetRoles(); appEm.refreshIndex(); @@@ -354,9 -333,9 +358,6 @@@ em.delete(appToDelete); } -- // delete the application's index -- EntityIndex ei = managerCache.getEntityIndex(new ApplicationScopeImpl(new SimpleId(applicationId, TYPE_APPLICATION))); -- ei.deleteIndex(); em.refreshIndex(); } @@@ -367,7 -346,7 +368,7 @@@ // remove old delete_appinfos Entity EntityManager em = getEntityManager(CpNamingUtils.SYSTEM_APP_ID); Query q = Query.fromQL(String.format("select * where applicationUuid = '%s'", applicationId.toString())); -- Results results = em.searchCollection( em.getApplicationRef(), "deleted_appinfos", q); ++ Results results = em.searchCollection(em.getApplicationRef(), "deleted_appinfos", q); Entity appToRestore = results.getEntity(); if ( appToRestore == null ) { @@@ -413,7 -392,7 +414,7 @@@ @Override public Map<String, UUID> getApplications() throws Exception { -- return getApplications( false ); ++ return getApplications(false); } @@@ -664,9 -644,9 +665,7 @@@ maybeCreateIndexes(); // system app -- for ( EntityIndex index : getManagementIndexes() ) { -- index.refresh(); -- } ++ entityIndex.refresh(); } @@@ -676,13 -656,13 +675,11 @@@ return; } -- for ( EntityIndex index : getManagementIndexes() ) { -- index.initializeIndex(); -- } ++ entityIndex.initializeIndex(); } -- private List<EntityIndex> getManagementIndexes() { ++ private List<ApplicationEntityIndex> getManagementIndexes() { return Arrays.asList( managerCache.getEntityIndex( new ApplicationScopeImpl( new SimpleId( CpNamingUtils.SYSTEM_APP_ID, "application" ) ) ), @@@ -764,7 -770,7 +760,6 @@@ @Override public void addIndex(final UUID applicationId,final String indexSuffix,final int shards,final int replicas, final String writeConsistency){ -- EntityIndex entityIndex = managerCache.getEntityIndex(CpNamingUtils.getApplicationScope(applicationId)); entityIndex.addIndex(indexSuffix, shards, replicas,writeConsistency); } http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b5ce0856/stack/core/src/main/java/org/apache/usergrid/corepersistence/ManagerCache.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b5ce0856/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRebuildIndexTest.java ---------------------------------------------------------------------- diff --cc stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRebuildIndexTest.java index f1f165d,f1f165d..c58750b --- a/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRebuildIndexTest.java +++ b/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRebuildIndexTest.java @@@ -25,6 -25,6 +25,7 @@@ import java.util.UUID import java.util.concurrent.TimeUnit; import org.apache.commons.lang.RandomStringUtils; ++import org.apache.usergrid.persistence.index.ApplicationEntityIndex; import org.junit.After; import org.junit.Before; import org.junit.Test; @@@ -175,17 -175,17 +176,17 @@@ public class PerformanceEntityRebuildIn // ----------------- delete the system and application indexes logger.debug("Deleting app index index"); -- //deleteIndex( CpNamingUtils.SYSTEM_APP_ID ); -- deleteIndex( em.getApplicationId() ); -- -- // ----------------- test that we can read them, should fail -- -- logger.debug("Reading data, should fail this time "); -- try { -- readData( em, "testTypes", entityCount, 0 ); -- fail("should have failed to read data"); -- -- } catch (Exception expected) {} ++// //deleteIndex( CpNamingUtils.SYSTEM_APP_ID ); ++// deleteIndex( em.getApplicationId() ); ++// ++// // ----------------- test that we can read them, should fail ++// ++// logger.debug("Reading data, should fail this time "); ++// try { ++// readData( em, "testTypes", entityCount, 0 ); ++// fail("should have failed to read data"); ++// ++// } catch (Exception expected) {} // ----------------- rebuild index for catherders only @@@ -313,20 -313,20 +314,20 @@@ logger.debug("Deleting app index and system app index"); -- deleteIndex( em.getApplicationId() ); -- -- // deleting sytem app index will interfere with other concurrently running tests -- //deleteIndex( CpNamingUtils.SYSTEM_APP_ID ); -- -- -- // ----------------- test that we can read them, should fail -- -- logger.debug("Reading data, should fail this time "); -- try { -- readData( em, "testTypes", entityCount, 3 ); -- fail("should have failed to read data"); -- -- } catch (Exception expected) {} ++// deleteIndex( em.getApplicationId() ); ++// ++// // deleting sytem app index will interfere with other concurrently running tests ++// //deleteIndex( CpNamingUtils.SYSTEM_APP_ID ); ++// ++// ++// // ----------------- test that we can read them, should fail ++// ++// logger.debug("Reading data, should fail this time "); ++// try { ++// readData( em, "testTypes", entityCount, 3 ); ++// fail("should have failed to read data"); ++// ++// } catch (Exception expected) {} // ----------------- rebuild index @@@ -374,21 -374,21 +375,20 @@@ readData( em, "testTypes", entityCount, 3 ); } -- /** -- * Delete index for all applications, just need the one to get started. -- */ -- private void deleteIndex( UUID appUuid ) { -- -- Injector injector = SpringResource.getInstance().getBean( Injector.class ); -- EntityIndexFactory eif = injector.getInstance( EntityIndexFactory.class ); -- -- Id appId = new SimpleId( appUuid, "application"); -- ApplicationScope scope = new ApplicationScopeImpl( appId ); -- EntityIndex ei = eif.createEntityIndex(scope); -- EsEntityIndexImpl eeii = (EsEntityIndexImpl)ei; -- -- eeii.deleteIndex(); -- } ++// /** ++// * Delete index for all applications, just need the one to get started. ++// */ ++// private void deleteIndex( UUID appUuid ) { ++// ++// Injector injector = SpringResource.getInstance().getBean( Injector.class ); ++// EntityIndexFactory eif = injector.getInstance( EntityIndexFactory.class ); ++// ++// Id appId = new SimpleId( appUuid, "application"); ++// ApplicationScope scope = new ApplicationScopeImpl( appId ); ++// ApplicationEntityIndex ei = eif.createApplicationEntityIndex(scope); ++// EsEntityIndexImpl eeii = (EsEntityIndexImpl)ei; ++// ++// } private int readData( EntityManager em, http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b5ce0856/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java ---------------------------------------------------------------------- diff --cc stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java index 1b3b115,521a4e7..854c976 --- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java +++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java @@@ -33,9 -33,9 +34,9 @@@ import java.util.concurrent.Future /** - * Provides indexing of Entities within a scope. + * Provides management operations for single index */ -public interface EntityIndex { +public interface EntityIndex extends CPManager { /** * This should ONLY ever be called once on application create. http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b5ce0856/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java ---------------------------------------------------------------------- diff --cc stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java index e66bd8d,ae5e3c3..d510a8d --- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java +++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java @@@ -49,16 -33,9 +33,14 @@@ import org.apache.usergrid.persistence. import org.apache.usergrid.persistence.model.util.UUIDGenerator; import org.elasticsearch.action.ActionFuture; - import org.elasticsearch.action.ActionListener; - import org.elasticsearch.action.ListenableActionFuture; - import org.elasticsearch.action.ShardOperationFailedException; + import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest; import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; +import org.elasticsearch.action.admin.cluster.tasks.PendingClusterTasksRequest; +import org.elasticsearch.action.admin.cluster.tasks.PendingClusterTasksResponse; +import org.elasticsearch.action.admin.indices.alias.IndicesAliasesResponse; +import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest; +import org.elasticsearch.action.admin.indices.alias.get.GetAliasesResponse; import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequestBuilder; import org.elasticsearch.action.admin.indices.alias.IndicesAliasesResponse; import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
