Damans227 commented on code in PR #14039:
URL: https://github.com/apache/cloudstack/pull/14039#discussion_r4284604286
##########
framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/impl/AsyncJobManagerImpl.java:
##########
@@ -701,58 +701,62 @@ private int getAndResetPendingSignals(AsyncJob job) {
return signals;
}
- private void executeQueueItem(SyncQueueItemVO item, boolean
fromPreviousSession) {
+ /**
+ * Dispatches the job behind a sync queue item.
+ * <p>
+ * TODO: stamping the executing management server id on the job can hit a
DB deadlock. As a
+ * temporary workaround, a failed update returns the item to the sync
queue so the job is retried
+ * on a later turn instead of failing. The proper fix is to remove the
deadlock at its source, in
+ * the locking and transaction around the sync_queue and async_job updates
on dispatch, after
+ * which this retry is no longer needed.
+ */
+ protected void executeQueueItem(SyncQueueItemVO item, boolean
fromPreviousSession) {
AsyncJobVO job = _jobDao.findById(item.getContentId());
- if (job != null) {
+ if (job == null) {
if (logger.isDebugEnabled()) {
- logger.debug("Schedule queued job-" + job.getId());
- }
-
- job.setSyncSource(item);
-
- //
- // TODO: a temporary solution to work-around DB deadlock situation
- //
- // to live with DB deadlocks, we will give a chance for job to be
rescheduled
- // in case of exceptions (most-likely DB deadlock exceptions)
- try {
- job.setExecutingMsid(getMsid());
- _jobDao.update(job.getId(), job);
- } catch (Exception e) {
- logger.warn("Unexpected exception while dispatching job-" +
item.getContentId(), e);
-
- try {
- _queueMgr.returnItem(item.getId());
- } catch (Throwable thr) {
- logger.error("Unexpected exception while returning job-" +
item.getContentId() + " to queue", thr);
- }
+ logger.debug("Unable to find related job for queue item: " +
item.toString());
}
+ _queueMgr.purgeItem(item.getId());
+ return;
+ }
- try {
- scheduleExecution(job);
- } catch (RejectedExecutionException e) {
- logger.warn("Execution for job-" + job.getId() + " is
rejected, return it to the queue for next turn");
+ if (logger.isDebugEnabled()) {
+ logger.debug("Schedule queued job-" + job.getId());
+ }
+ job.setSyncSource(item);
- try {
- _queueMgr.returnItem(item.getId());
- } catch (Exception e2) {
- logger.error("Unexpected exception while returning job-" +
item.getContentId() + " to queue", e2);
- }
+ try {
+ job.setExecutingMsid(getMsid());
+ _jobDao.update(job.getId(), job);
+ } catch (Exception e) {
+ logger.warn("Unexpected exception while dispatching job-" +
item.getContentId(), e);
+ returnItemToQueue(item);
Review Comment:
what happens if putting the item back in the queue also fails? that only
logs the error, so the job never runs and just sits there until cleanup fails
it about an hour later
##########
framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/impl/AsyncJobManagerImpl.java:
##########
@@ -701,58 +701,62 @@ private int getAndResetPendingSignals(AsyncJob job) {
return signals;
}
- private void executeQueueItem(SyncQueueItemVO item, boolean
fromPreviousSession) {
+ /**
+ * Dispatches the job behind a sync queue item.
+ * <p>
+ * TODO: stamping the executing management server id on the job can hit a
DB deadlock. As a
+ * temporary workaround, a failed update returns the item to the sync
queue so the job is retried
+ * on a later turn instead of failing. The proper fix is to remove the
deadlock at its source, in
+ * the locking and transaction around the sync_queue and async_job updates
on dispatch, after
+ * which this retry is no longer needed.
+ */
+ protected void executeQueueItem(SyncQueueItemVO item, boolean
fromPreviousSession) {
AsyncJobVO job = _jobDao.findById(item.getContentId());
- if (job != null) {
+ if (job == null) {
if (logger.isDebugEnabled()) {
- logger.debug("Schedule queued job-" + job.getId());
- }
-
- job.setSyncSource(item);
-
- //
- // TODO: a temporary solution to work-around DB deadlock situation
- //
- // to live with DB deadlocks, we will give a chance for job to be
rescheduled
- // in case of exceptions (most-likely DB deadlock exceptions)
- try {
- job.setExecutingMsid(getMsid());
- _jobDao.update(job.getId(), job);
- } catch (Exception e) {
- logger.warn("Unexpected exception while dispatching job-" +
item.getContentId(), e);
-
- try {
- _queueMgr.returnItem(item.getId());
- } catch (Throwable thr) {
- logger.error("Unexpected exception while returning job-" +
item.getContentId() + " to queue", thr);
- }
+ logger.debug("Unable to find related job for queue item: " +
item.toString());
}
+ _queueMgr.purgeItem(item.getId());
+ return;
+ }
- try {
- scheduleExecution(job);
- } catch (RejectedExecutionException e) {
- logger.warn("Execution for job-" + job.getId() + " is
rejected, return it to the queue for next turn");
+ if (logger.isDebugEnabled()) {
+ logger.debug("Schedule queued job-" + job.getId());
+ }
+ job.setSyncSource(item);
- try {
- _queueMgr.returnItem(item.getId());
- } catch (Exception e2) {
- logger.error("Unexpected exception while returning job-" +
item.getContentId() + " to queue", e2);
- }
+ try {
+ job.setExecutingMsid(getMsid());
+ _jobDao.update(job.getId(), job);
+ } catch (Exception e) {
+ logger.warn("Unexpected exception while dispatching job-" +
item.getContentId(), e);
+ returnItemToQueue(item);
+ return;
+ }
- try {
- job.setExecutingMsid(null);
- _jobDao.update(job.getId(), job);
- } catch (Exception e3) {
- logger.warn("Unexpected exception while update job-" +
item.getContentId() + " msid for bookkeeping");
- }
- }
+ try {
+ scheduleExecution(job);
+ } catch (RejectedExecutionException e) {
+ logger.warn("Execution for job-" + job.getId() + " is rejected,
return it to the queue for next turn");
+ returnItemToQueue(item);
+ clearExecutingMsid(job, item);
Review Comment:
should we clear the server id before putting the item back? another server
could pick it up in between and we would wipe its id
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
