Repository: cloudstack Updated Branches: refs/heads/master dd4f6bcaf -> 7cdb67dcf
CLOUDSTACK-7589: VM not Starting and always stuck in Stopped state after management server restarts. Project: http://git-wip-us.apache.org/repos/asf/cloudstack/repo Commit: http://git-wip-us.apache.org/repos/asf/cloudstack/commit/7cdb67dc Tree: http://git-wip-us.apache.org/repos/asf/cloudstack/tree/7cdb67dc Diff: http://git-wip-us.apache.org/repos/asf/cloudstack/diff/7cdb67dc Branch: refs/heads/master Commit: 7cdb67dcf1ec4158ec0ab4c2fa868cc63121bbb5 Parents: dd4f6bc Author: Min Chen <[email protected]> Authored: Fri Sep 19 15:12:09 2014 -0700 Committer: Min Chen <[email protected]> Committed: Fri Sep 19 15:12:31 2014 -0700 ---------------------------------------------------------------------- .../jobs/impl/AsyncJobManagerImpl.java | 57 +++++++------------- .../framework/jobs/impl/SyncQueueManager.java | 2 + .../jobs/impl/SyncQueueManagerImpl.java | 13 ++++- 3 files changed, 32 insertions(+), 40 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cloudstack/blob/7cdb67dc/framework/jobs/src/org/apache/cloudstack/framework/jobs/impl/AsyncJobManagerImpl.java ---------------------------------------------------------------------- diff --git a/framework/jobs/src/org/apache/cloudstack/framework/jobs/impl/AsyncJobManagerImpl.java b/framework/jobs/src/org/apache/cloudstack/framework/jobs/impl/AsyncJobManagerImpl.java index 7d374da..4c4d3c2 100644 --- a/framework/jobs/src/org/apache/cloudstack/framework/jobs/impl/AsyncJobManagerImpl.java +++ b/framework/jobs/src/org/apache/cloudstack/framework/jobs/impl/AsyncJobManagerImpl.java @@ -835,24 +835,6 @@ public class AsyncJobManagerImpl extends ManagerBase implements AsyncJobManager, return ManagementServerNode.getManagementServerId(); } - private void cleanupPendingJobs(List<SyncQueueItemVO> l) { - for (SyncQueueItemVO item : l) { - if (s_logger.isInfoEnabled()) { - s_logger.info("Discard left-over queue item: " + item.toString()); - } - - String contentType = item.getContentType(); - if (contentType != null && contentType.equalsIgnoreCase(SyncQueueItem.AsyncJobContentType)) { - Long jobId = item.getContentId(); - if (jobId != null) { - s_logger.warn("Mark job as failed as its correspoding queue-item has been discarded. job id: " + jobId); - completeAsyncJob(jobId, JobInfo.Status.FAILED, 0, "Execution was cancelled because of server shutdown"); - } - } - _queueMgr.purgeItem(item.getId()); - } - } - @DB protected List<Long> wakeupByJoinedJobCompletion(long joinedJobId) { SearchCriteria<Long> joinJobSC = JoinJobSearch.create("joinJobId", joinedJobId); @@ -967,6 +949,22 @@ public class AsyncJobManagerImpl extends ManagerBase implements AsyncJobManager, return true; } + private void cleanupLeftOverJobs(final long msid) { + try { + Transaction.execute(new TransactionCallbackNoReturn() { + @Override + public void doInTransactionWithoutResult(TransactionStatus status) { + // purge sync queue item running on this ms node + _queueMgr.cleanupActiveQueueItems(msid, true); + // reset job status for all jobs running on this ms node + _jobDao.resetJobProcess(msid, ApiErrorCode.INTERNAL_ERROR.getHttpCode(), "job cancelled because of management server restart or shutdown"); + } + }); + } catch (Throwable e) { + s_logger.warn("Unexpected exception in cleaning up left over jobs for mamagement server node " + msid, e); + } + } + @Override public void onManagementNodeJoined(List<? extends ManagementServerHost> nodeList, long selfNodeId) { } @@ -974,18 +972,7 @@ public class AsyncJobManagerImpl extends ManagerBase implements AsyncJobManager, @Override public void onManagementNodeLeft(List<? extends ManagementServerHost> nodeList, long selfNodeId) { for (final ManagementServerHost msHost : nodeList) { - try { - Transaction.execute(new TransactionCallbackNoReturn() { - @Override - public void doInTransactionWithoutResult(TransactionStatus status) { - List<SyncQueueItemVO> items = _queueMgr.getActiveQueueItems(msHost.getId(), true); - cleanupPendingJobs(items); - _jobDao.resetJobProcess(msHost.getId(), ApiErrorCode.INTERNAL_ERROR.getHttpCode(), "job cancelled because of management server restart"); - } - }); - } catch (Throwable e) { - s_logger.warn("Unexpected exception ", e); - } + cleanupLeftOverJobs(msHost.getId()); } } @@ -995,15 +982,7 @@ public class AsyncJobManagerImpl extends ManagerBase implements AsyncJobManager, @Override public boolean start() { - try { - _jobDao.cleanupPseduoJobs(getMsid()); - - List<SyncQueueItemVO> l = _queueMgr.getActiveQueueItems(getMsid(), false); - cleanupPendingJobs(l); - _jobDao.resetJobProcess(getMsid(), ApiErrorCode.INTERNAL_ERROR.getHttpCode(), "job cancelled because of management server restart"); - } catch (Throwable e) { - s_logger.error("Unexpected exception " + e.getMessage(), e); - } + cleanupLeftOverJobs(getMsid()); _heartbeatScheduler.scheduleAtFixedRate(getHeartbeatTask(), HEARTBEAT_INTERVAL, HEARTBEAT_INTERVAL, TimeUnit.MILLISECONDS); _heartbeatScheduler.scheduleAtFixedRate(getGCTask(), GC_INTERVAL, GC_INTERVAL, TimeUnit.MILLISECONDS); http://git-wip-us.apache.org/repos/asf/cloudstack/blob/7cdb67dc/framework/jobs/src/org/apache/cloudstack/framework/jobs/impl/SyncQueueManager.java ---------------------------------------------------------------------- diff --git a/framework/jobs/src/org/apache/cloudstack/framework/jobs/impl/SyncQueueManager.java b/framework/jobs/src/org/apache/cloudstack/framework/jobs/impl/SyncQueueManager.java index b521ffe..32d8464 100644 --- a/framework/jobs/src/org/apache/cloudstack/framework/jobs/impl/SyncQueueManager.java +++ b/framework/jobs/src/org/apache/cloudstack/framework/jobs/impl/SyncQueueManager.java @@ -36,4 +36,6 @@ public interface SyncQueueManager extends Manager { public List<SyncQueueItemVO> getBlockedQueueItems(long thresholdMs, boolean exclusive); void purgeAsyncJobQueueItemId(long asyncJobId); + + public void cleanupActiveQueueItems(Long msid, boolean exclusive); } http://git-wip-us.apache.org/repos/asf/cloudstack/blob/7cdb67dc/framework/jobs/src/org/apache/cloudstack/framework/jobs/impl/SyncQueueManagerImpl.java ---------------------------------------------------------------------- diff --git a/framework/jobs/src/org/apache/cloudstack/framework/jobs/impl/SyncQueueManagerImpl.java b/framework/jobs/src/org/apache/cloudstack/framework/jobs/impl/SyncQueueManagerImpl.java index 5160e05..1cfec4d 100644 --- a/framework/jobs/src/org/apache/cloudstack/framework/jobs/impl/SyncQueueManagerImpl.java +++ b/framework/jobs/src/org/apache/cloudstack/framework/jobs/impl/SyncQueueManagerImpl.java @@ -26,7 +26,6 @@ import org.apache.log4j.Logger; import org.apache.cloudstack.framework.jobs.dao.SyncQueueDao; import org.apache.cloudstack.framework.jobs.dao.SyncQueueItemDao; - import com.cloud.utils.DateUtil; import com.cloud.utils.component.ManagerBase; import com.cloud.utils.db.DB; @@ -260,4 +259,16 @@ public class SyncQueueManagerImpl extends ManagerBase implements SyncQueueManage purgeItem(itemId); } } + + @Override + public void cleanupActiveQueueItems(Long msid, boolean exclusive) { + List<SyncQueueItemVO> l = getActiveQueueItems(msid, false); + for (SyncQueueItemVO item : l) { + if (s_logger.isInfoEnabled()) { + s_logger.info("Discard left-over queue item: " + item.toString()); + } + purgeItem(item.getId()); + } + } + }
