Merge branch 'release-2.1.1' of https://git-wip-us.apache.org/repos/asf/usergrid into usergrid-1268-akka-211
# Conflicts: # stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java # stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java # stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java # stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/CollectionsResourceIT.java Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/a93cad8f Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/a93cad8f Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/a93cad8f Branch: refs/heads/release-2.1.1 Commit: a93cad8fae58679fcbfde9727aa675962f1347d4 Parents: 324ef35 3df0779 Author: Dave Johnson <[email protected]> Authored: Tue Jun 14 13:14:34 2016 -0400 Committer: Dave Johnson <[email protected]> Committed: Tue Jun 14 13:14:34 2016 -0400 ---------------------------------------------------------------------- .../main/resources/usergrid-default.properties | 3 +- .../corepersistence/CpEntityManager.java | 2 +- .../corepersistence/CpEntityManagerFactory.java | 244 +++++++++++++++---- .../corepersistence/EntityManagerFig.java | 2 +- .../asyncevents/AsyncEventService.java | 6 +- .../asyncevents/AsyncEventServiceImpl.java | 11 +- .../asyncevents/EventBuilder.java | 6 +- .../asyncevents/EventBuilderImpl.java | 46 ++-- .../model/DeIndexOldVersionsEvent.java | 12 +- .../corepersistence/index/IndexService.java | 23 +- .../corepersistence/index/IndexServiceImpl.java | 74 +++--- .../pipeline/cursor/CursorSerializerUtil.java | 9 - .../pipeline/cursor/RequestCursor.java | 29 ++- .../read/search/CandidateEntityFilter.java | 94 ++++++- .../apache/usergrid/locking/LockManager.java | 5 + .../locking/cassandra/AstyanaxLockImpl.java | 2 +- .../cassandra/AstyanaxLockManagerImpl.java | 113 +++++---- .../locking/noop/NoOpLockManagerImpl.java | 5 + .../persistence/entities/Notification.java | 38 ++- .../usergrid/persistence/CoreSchemaManager.java | 8 +- .../apache/usergrid/persistence/IndexIT.java | 98 ++++++++ .../test/resources/usergrid-test-context.xml | 1 + .../collection/EntityCollectionManager.java | 10 +- .../exception/CollectionRuntimeException.java | 11 + .../impl/EntityCollectionManagerImpl.java | 40 ++- .../serialization/SerializationFig.java | 5 + .../serialization/impl/LogEntryIterator.java | 128 ++++++++++ .../core/astyanax/CassandraCluster.java | 24 +- .../persistence/core/astyanax/CassandraFig.java | 18 +- .../model/field/value/EntityObject.java | 15 +- .../usergrid/persistence/index/EntityIndex.java | 5 +- .../usergrid/persistence/index/IndexFig.java | 2 +- .../index/impl/EsEntityIndexImpl.java | 64 ++--- .../persistence/index/impl/EntityIndexTest.java | 41 ---- .../security/SecuredResourceFilterFactory.java | 2 +- .../collection/CollectionsResourceIT.java | 47 ++++ .../queries/SelectMappingsQueryTest.java | 168 +++++++++++++ .../resource/endpoints/CollectionEndpoint.java | 24 +- .../test/resources/usergrid-test-context.xml | 2 +- .../usergrid/security/shiro/ShiroCache.java | 80 +++--- .../usergrid/services/ServiceManager.java | 8 + .../services/ServiceManagerFactory.java | 12 + .../impl/ApplicationQueueManagerImpl.java | 52 +++- .../AbstractServiceNotificationIT.java | 26 +- .../test/resources/usergrid-test-context.xml | 1 + 45 files changed, 1232 insertions(+), 384 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/usergrid/blob/a93cad8f/stack/config/src/main/resources/usergrid-default.properties ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/usergrid/blob/a93cad8f/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/usergrid/blob/a93cad8f/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java ---------------------------------------------------------------------- diff --cc stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java index 926cc0a,18df205..090400f --- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java +++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java @@@ -16,24 -16,17 +16,17 @@@ package org.apache.usergrid.corepersistence; - import java.util.HashMap; - import java.util.Map; - import java.util.Set; - import java.util.TreeMap; - import java.util.UUID; - - import org.apache.usergrid.persistence.collection.uniquevalues.AkkaFig; - import org.apache.usergrid.persistence.collection.uniquevalues.UniqueValuesService; - import org.slf4j.Logger; - import org.slf4j.LoggerFactory; - import org.springframework.beans.BeansException; - import org.springframework.context.ApplicationContext; - import org.springframework.context.ApplicationContextAware; - + import com.google.common.base.Optional; + import com.google.common.cache.CacheBuilder; + import com.google.common.cache.CacheLoader; + import com.google.common.cache.LoadingCache; + import com.google.common.util.concurrent.UncheckedExecutionException; + import com.google.inject.Injector; + import com.google.inject.Key; + import com.google.inject.TypeLiteral; import org.apache.commons.lang.StringUtils; - import org.apache.usergrid.corepersistence.asyncevents.AsyncEventService; -import org.apache.usergrid.corepersistence.index.IndexSchemaCacheFactory; +import org.apache.usergrid.corepersistence.index.CollectionSettingsCacheFactory; import org.apache.usergrid.corepersistence.index.ReIndexRequestBuilder; import org.apache.usergrid.corepersistence.index.ReIndexService; import org.apache.usergrid.corepersistence.service.CollectionService; @@@ -53,7 -39,8 +39,10 @@@ import org.apache.usergrid.persistence. import org.apache.usergrid.persistence.cassandra.CounterUtils; import org.apache.usergrid.persistence.cassandra.Setup; import org.apache.usergrid.persistence.collection.EntityCollectionManager; + import org.apache.usergrid.persistence.collection.exception.CollectionRuntimeException; import org.apache.usergrid.persistence.collection.serialization.impl.migration.EntityIdScope; ++import org.apache.usergrid.persistence.collection.uniquevalues.AkkaFig; ++import org.apache.usergrid.persistence.collection.uniquevalues.UniqueValuesService; import org.apache.usergrid.persistence.core.metrics.MetricsFactory; import org.apache.usergrid.persistence.core.migration.data.MigrationDataProvider; import org.apache.usergrid.persistence.core.scope.ApplicationScope; @@@ -104,16 -84,17 +86,16 @@@ public class CpEntityManagerFactory imp private Setup setup = null; + EntityManager managementAppEntityManager = null; + // cache of already instantiated entity managers - private LoadingCache<UUID, EntityManager> entityManagers - = CacheBuilder.newBuilder().maximumSize(100).build(new CacheLoader<UUID, EntityManager>() { - public EntityManager load(UUID appId) { // no checked exception - return _getEntityManager(appId); - } - }); + private final String ENTITY_MANAGER_CACHE_SIZE = "entity.manager.cache.size"; + private final LoadingCache<UUID, EntityManager> entityManagers; private final ApplicationIdCache applicationIdCache; - //private final IndexSchemaCache indexSchemaCache; + Application managementApp = null; + private ManagerCache managerCache; private CassandraService cassandraService; @@@ -125,11 -106,15 +107,16 @@@ private final CollectionService collectionService; private final ConnectionService connectionService; private final GraphManagerFactory graphManagerFactory; - private final IndexSchemaCacheFactory indexSchemaCacheFactory; + private final CollectionSettingsCacheFactory collectionSettingsCacheFactory; + private UniqueValuesService uniqueValuesService; + private final LockManager lockManager; - public CpEntityManagerFactory( final CassandraService cassandraService, final CounterUtils counterUtils, - final Injector injector ) { + public static final String MANAGEMENT_APP_INIT_MAXRETRIES= "management.app.init.max-retries"; + public static final String MANAGEMENT_APP_INIT_INTERVAL = "management.app.init.interval"; + + + public CpEntityManagerFactory( + final CassandraService cassandraService, final CounterUtils counterUtils, final Injector injector ) { this.cassandraService = cassandraService; this.counterUtils = counterUtils; @@@ -142,24 -127,162 +129,175 @@@ this.graphManagerFactory = injector.getInstance( GraphManagerFactory.class ); this.collectionService = injector.getInstance( CollectionService.class ); this.connectionService = injector.getInstance( ConnectionService.class ); - this.indexSchemaCacheFactory = injector.getInstance( IndexSchemaCacheFactory.class ); + + this.collectionSettingsCacheFactory = injector.getInstance( CollectionSettingsCacheFactory.class ); + + AkkaFig akkaFig = injector.getInstance( AkkaFig.class ); + + // this line always needs to be last due to the temporary cicular dependency until spring is removed + this.applicationIdCache = injector.getInstance(ApplicationIdCacheFactory.class).getInstance( + getManagementEntityManager() ); + + if ( akkaFig.getAkkaEnabled() ) { + try { + this.uniqueValuesService = injector.getInstance( UniqueValuesService.class ); + this.uniqueValuesService.start(); + } catch (Throwable t) { + logger.error("Error starting Akka", t); + throw t; + } + } + this.lockManager = injector.getInstance( LockManager.class ); + + Properties properties = cassandraService.getProperties(); + + entityManagers = createEntityManagerCache( properties ); + + checkManagementApp( properties ); - - // this line always needs to be last due to the temporary circular dependency until spring is removed - applicationIdCache = injector.getInstance(ApplicationIdCacheFactory.class) - .getInstance( getManagementEntityManager() ); + } + + + private LoadingCache<UUID, EntityManager> createEntityManagerCache(Properties properties) { + + int entityManagerCacheSize = 100; + try { + entityManagerCacheSize = Integer.parseInt( properties.getProperty( ENTITY_MANAGER_CACHE_SIZE, "100" )); + } catch ( Exception e ) { + logger.error("Error parsing " + ENTITY_MANAGER_CACHE_SIZE + ". Will use " + entityManagerCacheSize, e ); + } + + return CacheBuilder.newBuilder() + .maximumSize(entityManagerCacheSize) + .build(new CacheLoader<UUID, EntityManager>() { + + public EntityManager load( UUID appId ) { // no checked exception + + // create new entity manager and pre-fetch its application + EntityManager entityManager = _getEntityManager( appId ); + Application app = null; + Throwable throwable = null; + try { + app = entityManager.getApplication(); + } catch (Throwable t) { + throwable = t; + } + + // the management app is a special case + if ( CpNamingUtils.MANAGEMENT_APPLICATION_ID.equals( appId ) ) { + + if ( app != null ) { + // we successfully fetched up the management app, cache it for a rainy day + managementAppEntityManager = entityManager; + + } else if ( managementAppEntityManager != null ) { + // failed to fetch management app, use cached one + entityManager = managementAppEntityManager; + logger.error("Failed to fetch management app"); + } + } + + // missing keyspace means we have not done bootstrap yet + final boolean isBootstrapping; + if ( throwable instanceof CollectionRuntimeException ) { + CollectionRuntimeException cre = (CollectionRuntimeException) throwable; + isBootstrapping = cre.isBootstrapping(); + } else { + isBootstrapping = false; + } + + // work around for https://issues.apache.org/jira/browse/USERGRID-1291 + // throw exception so that we do not cache + // TODO: determine how application name can intermittently be null + if ( app != null && app.getName() == null ) { + throw new RuntimeException( "Name is null for application " + appId, throwable ); + } + + if ( app == null && !isBootstrapping ) { + throw new RuntimeException( "Error getting application " + appId, throwable ); + + } // else keyspace is missing because setup/bootstrap not done yet + + return entityManager; + } + }); + } + + + private void checkManagementApp(Properties properties) { + + int maxRetries = 100; + try { + maxRetries = Integer.parseInt( properties.getProperty( MANAGEMENT_APP_INIT_MAXRETRIES, "100" )); + + } catch ( Exception e ) { + logger.error("Error parsing " + MANAGEMENT_APP_INIT_MAXRETRIES + ". Will use " + maxRetries, e ); + } + + int interval = 1000; + try { + interval = Integer.parseInt( properties.getProperty( MANAGEMENT_APP_INIT_INTERVAL, "1000" )); + + } catch ( Exception e ) { + logger.error("Error parsing " + MANAGEMENT_APP_INIT_INTERVAL + ". Will use " + maxRetries, e ); + } + + // hold up construction until we can access the management app + int retries = 0; + boolean managementAppFound = false; + boolean bootstrapping = false; + Set<Class> seenBefore = new HashSet<>(10); + while ( !managementAppFound && retries++ < maxRetries ) { + try { + // bypass entity manager cache and get managementApp + managementApp = _getEntityManager( getManagementAppId() ).getApplication(); + managementAppFound = true; + + } catch ( Throwable t ) { + + if ( t instanceof CollectionRuntimeException ) { + CollectionRuntimeException cre = (CollectionRuntimeException)t; + if ( cre.isBootstrapping() ) { + // we're bootstrapping, ignore this and continue + bootstrapping = true; + break; + } + } + Throwable cause = t; + + // there was an error, be as informative as possible + StringBuilder sb = new StringBuilder(); + sb.append(retries).append(": Error ("); + + if ( t instanceof UncheckedExecutionException ) { + UncheckedExecutionException uee = (UncheckedExecutionException)t; + if ( uee.getCause() instanceof RuntimeException ) { + cause = uee.getCause().getCause(); + sb.append(cause.getClass().getSimpleName()).append(") ") + .append(uee.getCause().getMessage()); + } else { + cause = uee.getCause(); + sb.append(cause.getClass().getSimpleName()).append(") ").append(t.getMessage()); + } + } else { + sb.append(t.getCause().getClass().getSimpleName()).append(") ").append(t.getMessage()); + } + + String msg = sb.toString(); + if ( !seenBefore.contains( cause.getClass() ) ) { + logger.error( msg, t); + } else { + logger.error(msg); + } + seenBefore.add( cause.getClass() ); + + try { Thread.sleep( interval ); } catch (InterruptedException ignored) {} + } + } + + if ( !managementAppFound && !bootstrapping ) { + // exception here will prevent WAR from being deployed + throw new RuntimeException( "Unable to get management app after " + retries + " retries" ); + } } http://git-wip-us.apache.org/repos/asf/usergrid/blob/a93cad8f/stack/core/src/main/java/org/apache/usergrid/corepersistence/EntityManagerFig.java ---------------------------------------------------------------------- diff --cc stack/core/src/main/java/org/apache/usergrid/corepersistence/EntityManagerFig.java index 46c7a1d,655a968..872ffbb --- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/EntityManagerFig.java +++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/EntityManagerFig.java @@@ -39,15 -39,6 +39,15 @@@ public interface EntityManagerFig exten int sleep(); @Key( "usergrid.entityManager.enable_deindex_on_update" ) - @Default( "false" ) + @Default( "true" ) boolean getDeindexOnUpdate(); + + /** + * Comma-separated list of one or more Amazon regions to use if multiregion + * is set to true. + */ + @Key( "usergrid.queue.regionList" ) + @Default("us-east-1") + String getRegionList(); + } http://git-wip-us.apache.org/repos/asf/usergrid/blob/a93cad8f/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java ---------------------------------------------------------------------- diff --cc stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java index ab2caa3,54b18bb..d371b8a --- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java +++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java @@@ -43,13 -51,14 +43,12 @@@ import org.apache.usergrid.persistence. import org.apache.usergrid.persistence.model.entity.Id; import org.apache.usergrid.persistence.model.entity.SimpleId; import org.apache.usergrid.utils.InflectionUtils; - import org.apache.usergrid.utils.UUIDUtils; - -import com.codahale.metrics.Timer; -import com.google.common.base.Optional; -import com.google.inject.Inject; -import com.google.inject.Singleton; - +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import rx.Observable; +import java.util.*; + import static org.apache.usergrid.corepersistence.util.CpNamingUtils.*; import static org.apache.usergrid.persistence.Schema.TYPE_APPLICATION; http://git-wip-us.apache.org/repos/asf/usergrid/blob/a93cad8f/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/EntityCollectionManager.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/usergrid/blob/a93cad8f/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java ---------------------------------------------------------------------- diff --cc stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java index 544e7bb,fcf1c6b..09bd01b --- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java +++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java @@@ -27,7 -27,7 +27,8 @@@ import java.util.List import java.util.UUID; import com.netflix.astyanax.model.ConsistencyLevel; +import org.apache.usergrid.persistence.collection.uniquevalues.UniqueValuesService; + import org.apache.usergrid.persistence.collection.serialization.impl.LogEntryIterator; import org.slf4j.Logger; import org.slf4j.LoggerFactory; http://git-wip-us.apache.org/repos/asf/usergrid/blob/a93cad8f/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/CollectionsResourceIT.java ---------------------------------------------------------------------- diff --cc stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/CollectionsResourceIT.java index 48e125b,690cec2..f13ed46 --- a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/CollectionsResourceIT.java +++ b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/CollectionsResourceIT.java @@@ -17,24 -17,44 +17,26 @@@ package org.apache.usergrid.rest.applications.collection; -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.Iterator; -import java.util.LinkedHashMap; -import java.util.Map; -import java.util.Set; - -import javax.ws.rs.BadRequestException; -import javax.ws.rs.ClientErrorException; - -import org.junit.Ignore; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - +import com.fasterxml.jackson.databind.JsonNode; import org.apache.commons.lang.RandomStringUtils; - import org.apache.usergrid.persistence.Schema; import org.apache.usergrid.persistence.entities.Application; + import org.apache.usergrid.persistence.index.utils.UUIDUtils; import org.apache.usergrid.rest.test.resource.AbstractRestIT; -import org.apache.usergrid.rest.test.resource.model.ApiResponse; +import org.apache.usergrid.rest.test.resource.model.*; import org.apache.usergrid.rest.test.resource.model.Collection; -import org.apache.usergrid.rest.test.resource.model.Credentials; -import org.apache.usergrid.rest.test.resource.model.Entity; -import org.apache.usergrid.rest.test.resource.model.QueryParameters; -import org.apache.usergrid.rest.test.resource.model.Token; +import org.junit.Ignore; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -import com.fasterxml.jackson.databind.JsonNode; +import javax.ws.rs.BadRequestException; +import javax.ws.rs.ClientErrorException; +import java.io.IOException; +import java.util.*; + import static junit.framework.TestCase.assertNotNull; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotEquals; -import static org.junit.Assert.assertNotSame; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; +import static org.junit.Assert.*; /** @@@ -1003,58 -1103,48 +1005,103 @@@ public class CollectionsResourceIT exte assertEquals( 0, connectionsByQuery.getNumOfEntities() ); } + + @Test + public void testCollectionRegion() { + + // create collection with settings for index all + + String randomizer = RandomStringUtils.randomAlphanumeric(10); + String collectionName = "col_" + randomizer; + + app().collection( collectionName ).collection( "_settings" ) + .post( new Entity().chainPut( "fields", "all" ) ); + refreshIndex(); + + // get collection settings, should see no region + + Collection collection = app().collection( collectionName ).collection( "_settings" ).get(); + Map<String, Object> settings = (Map<String, Object>)collection.getResponse().getData(); + assertNull( settings.get( "region" )); + + // set collection region with bad region, expect error + + try { + app().collection( collectionName ).collection( "_settings" ) + .post( new Entity().chainPut( "region", "us-moon-1" ) ); + fail( "post should have failed"); + + } catch ( BadRequestException expected ) {} + + // set collection region with good region + + app().collection( collectionName ).collection( "_settings" ) + .post( new Entity().chainPut( "region", "us-east-1" ) ); + + // get collection settings see that we have a region + + collection = app().collection( collectionName ).collection( "_settings" ).get(); + settings = (Map<String, Object>)collection.getResponse().getData(); + assertNotNull( settings.get( "region" )); + assertEquals( "us-east-1", settings.get( "region" )); + + // unset the collection region + + app().collection( collectionName ).collection( "_settings" ) + .post( new Entity().chainPut( "region", "" ) ); + refreshIndex(); + + // get collection settings, should see no region + + collection = app().collection( collectionName ).collection( "_settings" ).get(); + settings = (Map<String, Object>)collection.getResponse().getData(); + assertNull( settings.get( "region" )); + + + } ++ ++ + @Test + public void testBeingAbleToRetreiveMigratedValues() throws Exception { + + + Entity notifier = new Entity().chainPut("name", "mynotifier").chainPut("provider", "noop"); + + ApiResponse notifierNode = this.pathResource(getOrgAppPath("notifier")).post(ApiResponse.class,notifier); + + // create user + + Map payloads = new HashMap<>( ); + payloads.put( "mynotifier","hello world" ); + + Map statistics = new HashMap<>( ); + statistics.put( "sent",1 ); + statistics.put( "errors",2 ); + + Entity payload = new Entity(); + payload.put("debug", false); + payload.put( "expectedCount",0 ); + payload.put( "finished",1438279671229L); + payload.put( "payloads",payloads); + payload.put( "priority","normal"); + payload.put( "state","FINISHED"); + payload.put( "statistics",statistics); + + + this.app().collection("notifications/"+ UUIDUtils.newTimeUUID()).post(payload ); + this.refreshIndex(); + + Collection user2 = this.app().collection("notifications").get(); + + assertEquals(1,user2.getNumOfEntities()); + + this.app().collection("notifications/"+ UUIDUtils.newTimeUUID()).put(null,payload ); + this.refreshIndex(); + + user2 = this.app().collection("notifications").get(); + + assertEquals(2,user2.getNumOfEntities()); + + } + - }
