Make retries and interval configurable
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/fcd00e83 Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/fcd00e83 Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/fcd00e83 Branch: refs/heads/release-2.1.1 Commit: fcd00e83b4febea1f395e7d8818bab8c38fa8bea Parents: 910811d Author: Dave Johnson <[email protected]> Authored: Wed May 25 18:30:27 2016 -0400 Committer: Dave Johnson <[email protected]> Committed: Wed May 25 18:30:27 2016 -0400 ---------------------------------------------------------------------- .../main/resources/usergrid-default.properties | 8 ++ .../corepersistence/CpEntityManagerFactory.java | 94 ++++++++++++-------- .../cassandra/AstyanaxLockManagerImpl.java | 5 +- .../persistence/core/astyanax/CassandraFig.java | 18 +++- 4 files changed, 86 insertions(+), 39 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/usergrid/blob/fcd00e83/stack/config/src/main/resources/usergrid-default.properties ---------------------------------------------------------------------- diff --git a/stack/config/src/main/resources/usergrid-default.properties b/stack/config/src/main/resources/usergrid-default.properties index 4f57cdd..1e22d6a 100644 --- a/stack/config/src/main/resources/usergrid-default.properties +++ b/stack/config/src/main/resources/usergrid-default.properties @@ -195,6 +195,12 @@ cassandra.lock.writecl=LOCAL_QUORUM # #cassandra.useSocketKeepalive=false +# Number of times to retry creation of lock keyspace and column family +cassandra.lock.init.retries = 100; + +# Interval between lock keyspace creation attempts (in milliseconds) +cassandra.lock.init.interval = 1000; + ##################### Async Threadpool Settings ##################### @@ -628,6 +634,8 @@ usergrid.auth.cache.inmemory.size=3000 # all (= in + out)' usergrid.rest.default-connection-param=all +# Number of times to retry attempt to access management app on startup +management.app.max.retries=100 ############################## Usergrid Testing ############################# http://git-wip-us.apache.org/repos/asf/usergrid/blob/fcd00e83/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java ---------------------------------------------------------------------- diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java index 84872aa..b6fbc2a 100644 --- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java +++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java @@ -23,6 +23,8 @@ import com.google.common.cache.LoadingCache; import com.google.inject.Injector; import com.google.inject.Key; import com.google.inject.TypeLiteral; +import com.netflix.astyanax.connectionpool.exceptions.BadRequestException; +import org.apache.cassandra.exceptions.InvalidRequestException; import org.apache.commons.lang.StringUtils; import org.apache.usergrid.corepersistence.asyncevents.AsyncEventService; import org.apache.usergrid.corepersistence.index.IndexSchemaCacheFactory; @@ -124,54 +126,79 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application this.connectionService = injector.getInstance( ConnectionService.class ); this.indexSchemaCacheFactory = injector.getInstance( IndexSchemaCacheFactory.class ); + Properties properties = cassandraService.getProperties(); + + entityManagers = createEntityManagerCache( properties ); + + checkManagementApp( properties ); + + // this line always needs to be last due to the temporary circular dependency until spring is removed + applicationIdCache = injector.getInstance(ApplicationIdCacheFactory.class) + .getInstance( getManagementEntityManager() ); + } + + + private LoadingCache<UUID, EntityManager> createEntityManagerCache(Properties properties) { + int entityManagerCacheSize = 100; try { - entityManagerCacheSize = Integer.parseInt( - cassandraService.getProperties().getProperty( ENTITY_MANAGER_CACHE_SIZE, "100" )); + entityManagerCacheSize = Integer.parseInt( properties.getProperty( ENTITY_MANAGER_CACHE_SIZE, "100" )); } catch ( Exception e ) { - logger.error("Error parsing " + ENTITY_MANAGER_CACHE_SIZE + " using " + entityManagerCacheSize, e ); + logger.error("Error parsing " + ENTITY_MANAGER_CACHE_SIZE + ". Will use " + entityManagerCacheSize, e ); } - entityManagers = CacheBuilder.newBuilder() + return CacheBuilder.newBuilder() .maximumSize(entityManagerCacheSize) .build(new CacheLoader<UUID, EntityManager>() { - public EntityManager load( UUID appId ) { // no checked exception + public EntityManager load( UUID appId ) { // no checked exception - // create new entity manager and pre-fetch its application - EntityManager entityManager = _getEntityManager( appId ); - Application app = null; - Exception exception = null; - try { - app = entityManager.getApplication(); - } catch (Exception e) { - exception = e; - } + // create new entity manager and pre-fetch its application + EntityManager entityManager = _getEntityManager( appId ); + Application app = null; + Throwable throwable = null; + try { + app = entityManager.getApplication(); + } catch (Throwable t) { + throwable = t; + } - // the management app is a special case + // the management app is a special case - if ( CpNamingUtils.MANAGEMENT_APPLICATION_ID.equals( appId ) ) { + if ( CpNamingUtils.MANAGEMENT_APPLICATION_ID.equals( appId ) ) { - if ( app != null ) { - // we successfully fetched up the management app, cache it for a rainy day - managementAppEntityManager = entityManager; + if ( app != null ) { + // we successfully fetched up the management app, cache it for a rainy day + managementAppEntityManager = entityManager; - } else if ( managementAppEntityManager != null ) { - // failed to fetch management app, use cached one - entityManager = managementAppEntityManager; + } else if ( managementAppEntityManager != null ) { + // failed to fetch management app, use cached one + entityManager = managementAppEntityManager; + } + } + + if ( app == null && throwable != null && throwable.getCause() instanceof BadRequestException) { + // probably means schema has not been created yet + } else { + throw new RuntimeException( "Error getting application " + appId, throwable ); } - } - if (app == null) { - throw new RuntimeException( "Error getting application " + appId, exception ); + return entityManager; } + }); + } - return entityManager; - } - }); + + private void checkManagementApp(Properties properties) { + int maxRetries = 100; + try { + maxRetries = Integer.parseInt( properties.getProperty( MANAGEMENT_APP_MAX_RETRIES, "100" )); + + } catch ( Exception e ) { + logger.error("Error parsing " + MANAGEMENT_APP_MAX_RETRIES + ". Will use " + maxRetries, e ); + } // hold up construction until we can access the management app - int maxRetries = 1000; int retries = 0; boolean managementAppFound = false; while ( !managementAppFound && retries++ < maxRetries ) { @@ -186,6 +213,7 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application } else { logger.error(msg); } + try { Thread.sleep( 1000 ); } catch (InterruptedException ignored) {} } } @@ -193,10 +221,6 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application // exception here will prevent WAR from being deployed throw new RuntimeException( "Unable to get management app after " + retries + " retries" ); } - - // this line always needs to be last due to the temporary circular dependency until spring is removed - applicationIdCache = injector.getInstance(ApplicationIdCacheFactory.class) - .getInstance( getManagementEntityManager() ); } @@ -240,8 +264,8 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application try { return entityManagers.get( applicationId ); } - catch ( Exception ex ) { - logger.error("Error getting oldAppInfo manager", ex); + catch ( Throwable t ) { + logger.error("Error getting entity manager", t); } return _getEntityManager(applicationId); } http://git-wip-us.apache.org/repos/asf/usergrid/blob/fcd00e83/stack/core/src/main/java/org/apache/usergrid/locking/cassandra/AstyanaxLockManagerImpl.java ---------------------------------------------------------------------- diff --git a/stack/core/src/main/java/org/apache/usergrid/locking/cassandra/AstyanaxLockManagerImpl.java b/stack/core/src/main/java/org/apache/usergrid/locking/cassandra/AstyanaxLockManagerImpl.java index 49ff52e..9bf6694 100644 --- a/stack/core/src/main/java/org/apache/usergrid/locking/cassandra/AstyanaxLockManagerImpl.java +++ b/stack/core/src/main/java/org/apache/usergrid/locking/cassandra/AstyanaxLockManagerImpl.java @@ -52,6 +52,7 @@ public class AstyanaxLockManagerImpl implements LockManager { private ColumnFamily columnFamily; private static final int MINIMUM_LOCK_EXPIRATION = 60000; // 1 minute + @Inject public AstyanaxLockManagerImpl(CassandraFig cassandraFig, CassandraCluster cassandraCluster ) throws ConnectionException { @@ -59,7 +60,7 @@ public class AstyanaxLockManagerImpl implements LockManager { this.cassandraFig = cassandraFig; // hold up construction until we can create the column family - int maxRetries = 1000; + int maxRetries = cassandraFig.getLockManagerInitRetries(); int retries = 0; boolean famReady = false; while ( !famReady && retries++ < maxRetries ) { @@ -76,7 +77,7 @@ public class AstyanaxLockManagerImpl implements LockManager { } else { logger.error( msg ); } - try { Thread.sleep(1000); } catch (InterruptedException ignored) {} + try { Thread.sleep( cassandraFig.getLockManagerInitInterval() ); } catch (InterruptedException ignored) {} } } http://git-wip-us.apache.org/repos/asf/usergrid/blob/fcd00e83/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/CassandraFig.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/CassandraFig.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/CassandraFig.java index b00eca8..d315561 100644 --- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/CassandraFig.java +++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/CassandraFig.java @@ -50,8 +50,8 @@ public interface CassandraFig extends GuicyFig { String LOCKS_CONNECTIONS = "cassandra.lock.connections"; String LOCKS_EXPIRATION = "cassandra.lock.expiration.milliseconds"; - - + String LOCK_MANAGER_INIT_RETRIES = "cassandra.lock.init.retries"; + String LOCK_MANAGER_INIT_INTERVAL = "cassandra.lock.init.interval"; // re-usable default values String DEFAULT_CONNECTION_POOLSIZE = "15"; @@ -180,4 +180,18 @@ public interface CassandraFig extends GuicyFig { @Default(DEFAULT_LOCKS_EXPIRATION) int getLocksExpiration(); + /** + * How many times to attempt lock keyspace and column family creation + */ + @Key( LOCK_MANAGER_INIT_RETRIES ) + @Default( "100" ) + int getLockManagerInitRetries(); + + /** + * Return the expiration that should be used for expiring a lock if it's not released + */ + @Key( LOCK_MANAGER_INIT_INTERVAL ) + @Default( "1000" ) + int getLockManagerInitInterval(); + }
