Author: kwall
Date: Wed Sep 24 13:02:57 2014
New Revision: 1627305
URL: http://svn.apache.org/r1627305
Log:
QPID-6111: [Java Broker] Ensure that when the REF is shutdown sufficient time
is allowed - bug fix used Math.min rather than Math.max.
Also:
* reduced the default envSetupTimeout from 15mins to 3mins.
* log a warning if environment re-creation takes longer than 25% of the allowed
timeout
* whilst restarting, recheck the REF state to avoid needlessly recreating the
environment if restart is no longer required (most likely owning to close)
Modified:
qpid/trunk/qpid/java/bdbstore/src/main/java/org/apache/qpid/server/store/berkeleydb/replication/ReplicatedEnvironmentFacade.java
Modified:
qpid/trunk/qpid/java/bdbstore/src/main/java/org/apache/qpid/server/store/berkeleydb/replication/ReplicatedEnvironmentFacade.java
URL:
http://svn.apache.org/viewvc/qpid/trunk/qpid/java/bdbstore/src/main/java/org/apache/qpid/server/store/berkeleydb/replication/ReplicatedEnvironmentFacade.java?rev=1627305&r1=1627304&r2=1627305&view=diff
==============================================================================
---
qpid/trunk/qpid/java/bdbstore/src/main/java/org/apache/qpid/server/store/berkeleydb/replication/ReplicatedEnvironmentFacade.java
(original)
+++
qpid/trunk/qpid/java/bdbstore/src/main/java/org/apache/qpid/server/store/berkeleydb/replication/ReplicatedEnvironmentFacade.java
Wed Sep 24 13:02:57 2014
@@ -157,7 +157,7 @@ public class ReplicatedEnvironmentFacade
/**
* Parameter decreased as the 10 h default may cause user confusion.
*/
- put(ReplicationConfig.ENV_SETUP_TIMEOUT, "15 min");
+ put(ReplicationConfig.ENV_SETUP_TIMEOUT, "180 s");
/**
* Parameter changed from default (off) to allow the Environment to
start in the
* UNKNOWN state when the majority is not available.
@@ -306,7 +306,7 @@ public class ReplicatedEnvironmentFacade
LOGGER.debug("Closing replicated environment facade for "
+ _prettyGroupNodeName + " current state is " + _state.get());
}
- long timeout = Math.min(_executorShutdownTimeout,
_envSetupTimeoutMillis);
+ long timeout = Math.max(_executorShutdownTimeout,
_envSetupTimeoutMillis);
shutdownAndAwaitExecutorService(_environmentJobExecutor,
timeout,
TimeUnit.MILLISECONDS);
@@ -407,23 +407,32 @@ public class ReplicatedEnvironmentFacade
@Override
public void run()
{
- for (int i = 0; i < _environmentRestartRetryLimit; i++)
+ int attemptNumber = 1;
+ boolean restarted = false;
+ while(_state.get() == State.RESTARTING && attemptNumber <=
_environmentRestartRetryLimit)
{
try
{
restartEnvironment();
+ restarted = true;
break;
}
catch(EnvironmentFailureException e)
{
- // log exception and try again
- LOGGER.warn("Unexpected failure on environment
restart. Restart iteration: " + i, e);
+ LOGGER.warn("Failure whilst trying to restart
environment (attempt number "
+ + attemptNumber + " of " +
_environmentRestartRetryLimit + ")", e);
}
catch (Exception e)
{
- LOGGER.error("Exception on environment restart",
e);
+ LOGGER.error("Fatal failure whilst trying to
restart environment", e);
break;
}
+ attemptNumber++;
+ }
+
+ if (!restarted)
+ {
+ LOGGER.warn("Failed to restart environment.");
}
}
});
@@ -565,8 +574,10 @@ public class ReplicatedEnvironmentFacade
{
if (LOGGER.isDebugEnabled())
{
- LOGGER.debug("Ignoring the state environment change event as
the environment facade for node '" + _prettyGroupNodeName
- + "' is in state " + _state.get());
+ LOGGER.debug("Ignoring the state environment change event as
the environment facade for node '"
+ + _prettyGroupNodeName
+ + "' is in state "
+ + _state.get());
}
}
}
@@ -1106,10 +1117,29 @@ public class ReplicatedEnvironmentFacade
return createEnvironment(environmentPathFile, envConfig,
replicationConfig);
}});
- long setUpTimeOutMillis =
extractEnvSetupTimeoutMillis(replicationConfig);
+ final long setUpTimeOutMillis =
extractEnvSetupTimeoutMillis(replicationConfig);
+ final long initialTimeOutMillis = Math.max(setUpTimeOutMillis / 4,
1000);
+ final long remainingTimeOutMillis = setUpTimeOutMillis -
initialTimeOutMillis;
try
{
- return environmentFuture.get(setUpTimeOutMillis,
TimeUnit.MILLISECONDS);
+ try
+ {
+ return environmentFuture.get(initialTimeOutMillis,
TimeUnit.MILLISECONDS);
+ }
+ catch (TimeoutException te)
+ {
+ if (remainingTimeOutMillis > 0)
+ {
+ LOGGER.warn("Slow replicated environment creation for " +
_prettyGroupNodeName
+ + ". Will continue to wait for further " +
remainingTimeOutMillis
+ + "ms. for environment creation to complete.");
+ return environmentFuture.get(remainingTimeOutMillis,
TimeUnit.MILLISECONDS);
+ }
+ else
+ {
+ throw te;
+ }
+ }
}
catch (InterruptedException e)
{
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
