Murtadha Hubail has submitted this change and it was merged. Change subject: [ASTERIXDB-2197][FAIL] Abort Job on Failures in NotifyTaskCompleteWork ......................................................................
[ASTERIXDB-2197][FAIL] Abort Job on Failures in NotifyTaskCompleteWork - user model changes: no - storage format changes: no - interface changes: no Details: - Abort the job if any unexpected failure happens on NotifyTaskCompleteWork to ensure that the job will not be waiting forever. Change-Id: I60c911c7aae872ee6b94e68efa53638207c0180d Reviewed-on: https://asterix-gerrit.ics.uci.edu/2218 Sonar-Qube: Jenkins <[email protected]> Tested-by: Jenkins <[email protected]> Contrib: Jenkins <[email protected]> Integration-Tests: Jenkins <[email protected]> Reviewed-by: Till Westmann <[email protected]> --- M hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/executor/JobExecutor.java M hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/TaskCompleteWork.java 2 files changed, 38 insertions(+), 37 deletions(-) Approvals: Anon. E. Moose #1000171: Till Westmann: Looks good to me, approved Jenkins: Verified; No violations found; ; Verified diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/executor/JobExecutor.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/executor/JobExecutor.java index 01201a6..ab7a3db 100644 --- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/executor/JobExecutor.java +++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/executor/JobExecutor.java @@ -642,27 +642,32 @@ return doomed; } - public void notifyTaskComplete(TaskAttempt ta) throws HyracksException { - TaskAttemptId taId = ta.getTaskAttemptId(); - TaskCluster tc = ta.getTask().getTaskCluster(); - TaskClusterAttempt lastAttempt = findLastTaskClusterAttempt(tc); - if (lastAttempt == null || taId.getAttempt() != lastAttempt.getAttempt()) { - LOGGER.warning( - "Ignoring task complete notification: " + taId + " -- Current last attempt = " + lastAttempt); - return; - } - TaskAttempt.TaskStatus taStatus = ta.getStatus(); - if (taStatus != TaskAttempt.TaskStatus.RUNNING) { - LOGGER.warning("Spurious task complete notification: " + taId + " Current state = " + taStatus); - return; - } - ta.setStatus(TaskAttempt.TaskStatus.COMPLETED, null); - ta.setEndTime(System.currentTimeMillis()); - if (lastAttempt.decrementPendingTasksCounter() == 0) { - lastAttempt.setStatus(TaskClusterAttempt.TaskClusterStatus.COMPLETED); - lastAttempt.setEndTime(System.currentTimeMillis()); - inProgressTaskClusters.remove(tc); - startRunnableActivityClusters(); + public void notifyTaskComplete(TaskAttempt ta) { + try { + TaskAttemptId taId = ta.getTaskAttemptId(); + TaskCluster tc = ta.getTask().getTaskCluster(); + TaskClusterAttempt lastAttempt = findLastTaskClusterAttempt(tc); + if (lastAttempt == null || taId.getAttempt() != lastAttempt.getAttempt()) { + LOGGER.warning(() -> "Ignoring task complete notification: " + taId + " -- Current last attempt = " + + lastAttempt); + return; + } + TaskAttempt.TaskStatus taStatus = ta.getStatus(); + if (taStatus != TaskAttempt.TaskStatus.RUNNING) { + LOGGER.warning(() -> "Spurious task complete notification: " + taId + " Current state = " + taStatus); + return; + } + ta.setStatus(TaskAttempt.TaskStatus.COMPLETED, null); + ta.setEndTime(System.currentTimeMillis()); + if (lastAttempt.decrementPendingTasksCounter() == 0) { + lastAttempt.setStatus(TaskClusterAttempt.TaskClusterStatus.COMPLETED); + lastAttempt.setEndTime(System.currentTimeMillis()); + inProgressTaskClusters.remove(tc); + startRunnableActivityClusters(); + } + } catch (Exception e) { + LOGGER.log(Level.SEVERE, e, () -> "Unexpected failure. Aborting job " + jobRun.getJobId()); + abortJob(Collections.singletonList(e), NoOpCallback.INSTANCE); } } @@ -724,6 +729,7 @@ ta -> HyracksException.create(ErrorCode.NODE_FAILED, ta.getNodeId())); startRunnableActivityClusters(); } catch (Exception e) { + LOGGER.log(Level.SEVERE, e, () -> "Unexpected failure. Aborting job " + jobRun.getJobId()); abortJob(Collections.singletonList(e), NoOpCallback.INSTANCE); } } diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/TaskCompleteWork.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/TaskCompleteWork.java index f4f2f52..e2f8b0d 100644 --- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/TaskCompleteWork.java +++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/TaskCompleteWork.java @@ -21,7 +21,6 @@ import java.util.Map; import org.apache.hyracks.api.dataflow.TaskAttemptId; -import org.apache.hyracks.api.exceptions.HyracksException; import org.apache.hyracks.api.job.JobId; import org.apache.hyracks.control.cc.ClusterControllerService; import org.apache.hyracks.control.cc.job.IJobManager; @@ -42,23 +41,19 @@ @Override protected void performEvent(TaskAttempt ta) { - try { - IJobManager jobManager = ccs.getJobManager(); - JobRun run = jobManager.get(jobId); - if (statistics != null) { - JobProfile jobProfile = run.getJobProfile(); - Map<String, JobletProfile> jobletProfiles = jobProfile.getJobletProfiles(); - JobletProfile jobletProfile = jobletProfiles.get(nodeId); - if (jobletProfile == null) { - jobletProfile = new JobletProfile(nodeId); - jobletProfiles.put(nodeId, jobletProfile); - } - jobletProfile.getTaskProfiles().put(taId, statistics); + IJobManager jobManager = ccs.getJobManager(); + JobRun run = jobManager.get(jobId); + if (statistics != null) { + JobProfile jobProfile = run.getJobProfile(); + Map<String, JobletProfile> jobletProfiles = jobProfile.getJobletProfiles(); + JobletProfile jobletProfile = jobletProfiles.get(nodeId); + if (jobletProfile == null) { + jobletProfile = new JobletProfile(nodeId); + jobletProfiles.put(nodeId, jobletProfile); } - run.getExecutor().notifyTaskComplete(ta); - } catch (HyracksException e) { - e.printStackTrace(); + jobletProfile.getTaskProfiles().put(taId, statistics); } + run.getExecutor().notifyTaskComplete(ta); } @Override -- To view, visit https://asterix-gerrit.ics.uci.edu/2218 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-MessageType: merged Gerrit-Change-Id: I60c911c7aae872ee6b94e68efa53638207c0180d Gerrit-PatchSet: 2 Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Owner: Murtadha Hubail <[email protected]> Gerrit-Reviewer: Anon. E. Moose #1000171 Gerrit-Reviewer: Jenkins <[email protected]> Gerrit-Reviewer: Michael Blow <[email protected]> Gerrit-Reviewer: Murtadha Hubail <[email protected]> Gerrit-Reviewer: Till Westmann <[email protected]> Gerrit-Reviewer: abdullah alamoudi <[email protected]>
