Move LockManager setup into normal setup regime, don't cache EntityManagers with null application names, and add logging to help debug issue USERGRID-1283.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/59c538aa Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/59c538aa Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/59c538aa Branch: refs/heads/master Commit: 59c538aa8a6becf90573c815e68347036f1aca44 Parents: 61a35a0 Author: Dave Johnson <[email protected]> Authored: Tue May 31 09:34:48 2016 -0400 Committer: Dave Johnson <[email protected]> Committed: Tue May 31 09:34:48 2016 -0400 ---------------------------------------------------------------------- .../corepersistence/CpEntityManagerFactory.java | 13 +++++- .../apache/usergrid/locking/LockManager.java | 5 +++ .../cassandra/AstyanaxLockManagerImpl.java | 42 ++++++-------------- .../locking/noop/NoOpLockManagerImpl.java | 5 +++ .../usergrid/services/ServiceManager.java | 8 ++++ .../services/ServiceManagerFactory.java | 12 ++++++ 6 files changed, 54 insertions(+), 31 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/usergrid/blob/59c538aa/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java ---------------------------------------------------------------------- diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java index ee28765..622944b 100644 --- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java +++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java @@ -33,6 +33,7 @@ import org.apache.usergrid.corepersistence.service.CollectionService; import org.apache.usergrid.corepersistence.service.ConnectionService; import org.apache.usergrid.corepersistence.util.CpNamingUtils; import org.apache.usergrid.exception.ConflictException; +import org.apache.usergrid.locking.LockManager; import org.apache.usergrid.persistence.*; import org.apache.usergrid.persistence.cassandra.CassandraService; import org.apache.usergrid.persistence.cassandra.CounterUtils; @@ -107,6 +108,7 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application private final ConnectionService connectionService; private final GraphManagerFactory graphManagerFactory; private final IndexSchemaCacheFactory indexSchemaCacheFactory; + private final LockManager lockManager; public static final String MANAGEMENT_APP_INIT_MAXRETRIES= "management.app.init.max-retries"; public static final String MANAGEMENT_APP_INIT_INTERVAL = "management.app.init.interval"; @@ -127,6 +129,7 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application this.collectionService = injector.getInstance( CollectionService.class ); this.connectionService = injector.getInstance( ConnectionService.class ); this.indexSchemaCacheFactory = injector.getInstance( IndexSchemaCacheFactory.class ); + this.lockManager = injector.getInstance( LockManager.class ); Properties properties = cassandraService.getProperties(); @@ -166,7 +169,6 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application } // the management app is a special case - if ( CpNamingUtils.MANAGEMENT_APPLICATION_ID.equals( appId ) ) { if ( app != null ) { @@ -179,6 +181,7 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application } } + // missing keyspace means we have not done bootstrap yet final boolean missingKeyspace; if ( throwable instanceof CollectionRuntimeException ) { CollectionRuntimeException cre = (CollectionRuntimeException) throwable; @@ -187,6 +190,13 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application missingKeyspace = false; } + // work around for https://issues.apache.org/jira/browse/USERGRID-1291 + // throw exception so that we do not cache + // TODO: determine how application name can intermittently be null + if ( app != null && app.getName() == null ) { + throw new RuntimeException( "Name is null for application " + appId, throwable ); + } + if ( app == null && !missingKeyspace ) { throw new RuntimeException( "Error getting application " + appId, throwable ); @@ -645,6 +655,7 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application @Override public void setup() throws Exception { getSetup().initSchema(); + lockManager.setup(); } http://git-wip-us.apache.org/repos/asf/usergrid/blob/59c538aa/stack/core/src/main/java/org/apache/usergrid/locking/LockManager.java ---------------------------------------------------------------------- diff --git a/stack/core/src/main/java/org/apache/usergrid/locking/LockManager.java b/stack/core/src/main/java/org/apache/usergrid/locking/LockManager.java index a2d37ab..4ed41d8 100644 --- a/stack/core/src/main/java/org/apache/usergrid/locking/LockManager.java +++ b/stack/core/src/main/java/org/apache/usergrid/locking/LockManager.java @@ -38,4 +38,9 @@ public interface LockManager { * @throws UGLockException if the lock cannot be acquired */ public Lock createLock( final UUID applicationId, final String... path ); + + /** + * Setup lock persistence mechanism. + */ + public void setup(); } http://git-wip-us.apache.org/repos/asf/usergrid/blob/59c538aa/stack/core/src/main/java/org/apache/usergrid/locking/cassandra/AstyanaxLockManagerImpl.java ---------------------------------------------------------------------- diff --git a/stack/core/src/main/java/org/apache/usergrid/locking/cassandra/AstyanaxLockManagerImpl.java b/stack/core/src/main/java/org/apache/usergrid/locking/cassandra/AstyanaxLockManagerImpl.java index 6acce47..90b9d57 100644 --- a/stack/core/src/main/java/org/apache/usergrid/locking/cassandra/AstyanaxLockManagerImpl.java +++ b/stack/core/src/main/java/org/apache/usergrid/locking/cassandra/AstyanaxLockManagerImpl.java @@ -50,6 +50,7 @@ public class AstyanaxLockManagerImpl implements LockManager { private final CassandraFig cassandraFig; + private final CassandraCluster cassandraCluster; private Keyspace keyspace; private ColumnFamily columnFamily; private static final int MINIMUM_LOCK_EXPIRATION = 60000; // 1 minute @@ -60,38 +61,19 @@ public class AstyanaxLockManagerImpl implements LockManager { CassandraCluster cassandraCluster ) throws ConnectionException { this.cassandraFig = cassandraFig; + this.cassandraCluster = cassandraCluster; + } - // hold up construction until we can create the column family - int maxRetries = cassandraFig.getLockManagerInitRetries(); - int retries = 0; - boolean famReady = false; - Set<Class> seenBefore = new HashSet<>(10); - while ( !famReady && retries++ < maxRetries ) { - try { - keyspace = cassandraCluster.getLocksKeyspace(); - createLocksKeyspace(); - columnFamily = createLocksColumnFamily(); - famReady = true; - - } catch ( Throwable t ) { - final String msg; - if ( t instanceof PoolTimeoutException || t instanceof NoAvailableHostsException) { - msg = retries + ": Cannot connect to Cassandra (" + t.getClass().getSimpleName() + ")"; - } else { - msg = retries + ": Error (" + t.getClass().getSimpleName() + ") tries=" + retries; - } - if ( !seenBefore.contains( t.getClass() ) ) { - logger.error( msg, t ); - } else { - logger.error( msg ); - } - seenBefore.add( t.getClass() ); - try { - Thread.sleep( cassandraFig.getLockManagerInitInterval() ); - } catch (InterruptedException ignored) {} - } - } + @Override + public void setup() { + try { + keyspace = cassandraCluster.getLocksKeyspace(); + createLocksKeyspace(); + columnFamily = createLocksColumnFamily(); + } catch (ConnectionException e) { + throw new RuntimeException( "Error setting up locks keyspace and column family", e ); + } } http://git-wip-us.apache.org/repos/asf/usergrid/blob/59c538aa/stack/core/src/main/java/org/apache/usergrid/locking/noop/NoOpLockManagerImpl.java ---------------------------------------------------------------------- diff --git a/stack/core/src/main/java/org/apache/usergrid/locking/noop/NoOpLockManagerImpl.java b/stack/core/src/main/java/org/apache/usergrid/locking/noop/NoOpLockManagerImpl.java index 6174890..ff14031 100644 --- a/stack/core/src/main/java/org/apache/usergrid/locking/noop/NoOpLockManagerImpl.java +++ b/stack/core/src/main/java/org/apache/usergrid/locking/noop/NoOpLockManagerImpl.java @@ -38,4 +38,9 @@ public class NoOpLockManagerImpl implements LockManager { public Lock createLock( UUID applicationId, String... path ) { return new NoOpLockImpl(); } + + @Override + public void setup() { + // no op + } } http://git-wip-us.apache.org/repos/asf/usergrid/blob/59c538aa/stack/services/src/main/java/org/apache/usergrid/services/ServiceManager.java ---------------------------------------------------------------------- diff --git a/stack/services/src/main/java/org/apache/usergrid/services/ServiceManager.java b/stack/services/src/main/java/org/apache/usergrid/services/ServiceManager.java index a9892f5..04e00e0 100644 --- a/stack/services/src/main/java/org/apache/usergrid/services/ServiceManager.java +++ b/stack/services/src/main/java/org/apache/usergrid/services/ServiceManager.java @@ -96,6 +96,14 @@ public class ServiceManager { this.qm = qm; this.properties = properties; + // additional logging to help debug https://issues.apache.org/jira/browse/USERGRID-1291 + if ( em == null ) { + logger.error("EntityManager is null"); + } + if ( qm == null ) { + logger.error("QueueManager is null"); + } + if ( em != null ) { try { application = em.getApplication(); http://git-wip-us.apache.org/repos/asf/usergrid/blob/59c538aa/stack/services/src/main/java/org/apache/usergrid/services/ServiceManagerFactory.java ---------------------------------------------------------------------- diff --git a/stack/services/src/main/java/org/apache/usergrid/services/ServiceManagerFactory.java b/stack/services/src/main/java/org/apache/usergrid/services/ServiceManagerFactory.java index 5274336..2425b95 100644 --- a/stack/services/src/main/java/org/apache/usergrid/services/ServiceManagerFactory.java +++ b/stack/services/src/main/java/org/apache/usergrid/services/ServiceManagerFactory.java @@ -23,6 +23,8 @@ import java.util.UUID; import com.google.inject.Injector; import org.apache.usergrid.locking.Lock; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.beans.BeansException; import org.springframework.context.ApplicationContext; import org.springframework.context.ApplicationContextAware; @@ -35,6 +37,7 @@ import org.apache.usergrid.persistence.EntityManagerFactory; public class ServiceManagerFactory implements ApplicationContextAware { + private static final Logger logger = LoggerFactory.getLogger( ServiceManagerFactory.class ); private ApplicationContext applicationContext; @@ -59,6 +62,15 @@ public class ServiceManagerFactory implements ApplicationContextAware { public ServiceManager getServiceManager( UUID applicationId ) { + + // additional logging to help debug https://issues.apache.org/jira/browse/USERGRID-1291 + if ( emf == null ) { + logger.error("EntityManagerFactory is null"); + } + if ( qmf == null ) { + logger.error("QueueManagerFactory is null"); + } + EntityManager em = null; if ( emf != null ) { em = emf.getEntityManager( applicationId );
