BUG-ID: CS-19196: Relax HA work termination rule on exceptions. It could cause premature HA termination due to unhandled exceptions. Reviewed-By:Anthony
Project: http://git-wip-us.apache.org/repos/asf/cloudstack/repo Commit: http://git-wip-us.apache.org/repos/asf/cloudstack/commit/3123c30f Tree: http://git-wip-us.apache.org/repos/asf/cloudstack/tree/3123c30f Diff: http://git-wip-us.apache.org/repos/asf/cloudstack/diff/3123c30f Branch: refs/heads/resize-root Commit: 3123c30f23ee0f089a554ec5ea50fac1c4731329 Parents: 5310e66 Author: Kelven Yang <[email protected]> Authored: Fri Mar 7 17:15:04 2014 -0800 Committer: Kelven Yang <[email protected]> Committed: Thu Mar 13 16:59:56 2014 -0700 ---------------------------------------------------------------------- server/src/com/cloud/ha/HighAvailabilityManagerImpl.java | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cloudstack/blob/3123c30f/server/src/com/cloud/ha/HighAvailabilityManagerImpl.java ---------------------------------------------------------------------- diff --git a/server/src/com/cloud/ha/HighAvailabilityManagerImpl.java b/server/src/com/cloud/ha/HighAvailabilityManagerImpl.java index 52100c8..48d998a 100755 --- a/server/src/com/cloud/ha/HighAvailabilityManagerImpl.java +++ b/server/src/com/cloud/ha/HighAvailabilityManagerImpl.java @@ -913,8 +913,14 @@ public class HighAvailabilityManagerImpl extends ManagerBase implements HighAvai work.setDateTaken(null); } } catch (Exception e) { - s_logger.error("Terminating " + work, e); - work.setStep(Step.Error); + s_logger.warn("Encountered unhandled exception during HA process, reschedule retry", e); + + long nextTime = (System.currentTimeMillis() >> 10) + _restartRetryInterval; + + s_logger.info("Rescheduling " + work + " to try again at " + new Date(nextTime << 10)); + work.setTimeToTry(nextTime); + work.setServerId(null); + work.setDateTaken(null); } _haDao.update(work.getId(), work); } catch (final Throwable th) {
