abdullah alamoudi has submitted this change and it was merged. Change subject: [NO ISSUE][RT] Improve logging in task related works ......................................................................
[NO ISSUE][RT] Improve logging in task related works Change-Id: Ia4e24a95aaac37b1d3d0d9a35266109ae0315293 Reviewed-on: https://asterix-gerrit.ics.uci.edu/2803 Sonar-Qube: Jenkins <[email protected]> Tested-by: Jenkins <[email protected]> Contrib: Jenkins <[email protected]> Integration-Tests: Jenkins <[email protected]> Reviewed-by: Till Westmann <[email protected]> --- M hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/job/JobManager.java M hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/Task.java M hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/work/NotifyTaskCompleteWork.java M hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/work/NotifyTaskFailureWork.java M hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/collectors/InputChannelFrameReader.java M hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/collectors/NonDeterministicChannelReader.java 6 files changed, 29 insertions(+), 14 deletions(-) Approvals: Anon. E. Moose #1000171: Till Westmann: Looks good to me, approved Jenkins: Verified; No violations found; ; Verified diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/job/JobManager.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/job/JobManager.java index b6b3e40..7dc636c 100644 --- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/job/JobManager.java +++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/job/JobManager.java @@ -221,17 +221,17 @@ JobId jobId = run.getJobId(); Throwable caughtException = null; CCServiceContext serviceCtx = ccs.getContext(); - if (serviceCtx != null) { - try { - serviceCtx.notifyJobFinish(jobId, run.getPendingStatus(), run.getPendingExceptions()); - } catch (Exception e) { - LOGGER.error("Exception notifying job finish {}", jobId, e); - caughtException = e; - } + try { + serviceCtx.notifyJobFinish(jobId, run.getPendingStatus(), run.getPendingExceptions()); + } catch (Exception e) { + LOGGER.error("Exception notifying job finish {}", jobId, e); + caughtException = e; } run.setStatus(run.getPendingStatus(), run.getPendingExceptions()); run.setEndTime(System.currentTimeMillis()); - activeRunMap.remove(jobId); + if (activeRunMap.remove(jobId) == null) { + LOGGER.warn("Job {} was not found running but is getting archived and capacity released", jobId); + } runMapArchive.put(jobId, run); runMapHistory.put(jobId, run.getExceptions()); diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/Task.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/Task.java index d7b930c..23a5abb 100644 --- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/Task.java +++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/Task.java @@ -290,7 +290,7 @@ .schedule(new NotifyTaskFailureWork(ncs, this, exceptions, joblet.getJobId(), taskAttemptId)); return; } - ct.setName(displayName + ":" + taskAttemptId + ":" + 0); + ct.setName(displayName + ":" + joblet.getJobId() + ":" + taskAttemptId + ":" + 0); try { Throwable operatorException = null; try { @@ -469,4 +469,10 @@ public boolean isCompleted() { return completed; } + + @Override + public String toString() { + return "{ \"class\" : \"" + getClass().getSimpleName() + "\", \"node\" : \"" + ncs.getId() + "\" \"jobId\" : \"" + + joblet.getJobId() + "\", \"taskId\" : \"" + taskAttemptId + "\" }"; + } } diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/work/NotifyTaskCompleteWork.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/work/NotifyTaskCompleteWork.java index 614a9e0..554c660 100644 --- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/work/NotifyTaskCompleteWork.java +++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/work/NotifyTaskCompleteWork.java @@ -51,6 +51,7 @@ @Override public String toString() { - return getClass().getSimpleName() + ":" + task.getTaskAttemptId(); + return getName() + ": [" + ncs.getId() + "[" + task.getJoblet().getJobId() + ":" + task.getTaskAttemptId() + + "]"; } } \ No newline at end of file diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/work/NotifyTaskFailureWork.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/work/NotifyTaskFailureWork.java index f0b68a0..ac80afa 100644 --- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/work/NotifyTaskFailureWork.java +++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/work/NotifyTaskFailureWork.java @@ -64,4 +64,9 @@ task.getJoblet().removeTask(task); } } + + @Override + public String toString() { + return getName() + ": [" + ncs.getId() + "[" + jobId + ":" + taskId + "]"; + } } diff --git a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/collectors/InputChannelFrameReader.java b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/collectors/InputChannelFrameReader.java index 0efed6f..bf9f575 100644 --- a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/collectors/InputChannelFrameReader.java +++ b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/collectors/InputChannelFrameReader.java @@ -27,8 +27,11 @@ import org.apache.hyracks.api.comm.IFrameReader; import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.dataflow.common.comm.util.FrameUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; public class InputChannelFrameReader implements IFrameReader, IInputChannelMonitor { + private static final Logger LOGGER = LogManager.getLogger(); private final IInputChannel channel; private int availableFrames; @@ -59,6 +62,7 @@ if (failed) { // Do not throw exception here to allow the root cause exception gets propagated to the master first. // Return false to allow the nextFrame(...) call to be a non-op. + LOGGER.warn("Sender failed.. returning silently"); return false; } if (availableFrames <= 0 && eos) { diff --git a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/collectors/NonDeterministicChannelReader.java b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/collectors/NonDeterministicChannelReader.java index b6f7cad..3c0a06b 100644 --- a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/collectors/NonDeterministicChannelReader.java +++ b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/collectors/NonDeterministicChannelReader.java @@ -106,6 +106,7 @@ return lastReadSender; } if (!failSenders.isEmpty()) { + LOGGER.warn("Sender failed.. returning silently"); // Do not throw exception here to allow the root cause exception gets propagated to the master first. // Return a negative value to allow the nextFrame(...) call to be a non-op. return -1; @@ -143,10 +144,8 @@ public synchronized void notifyFailure(IInputChannel channel) { PartitionId pid = (PartitionId) channel.getAttachment(); int senderIndex = pid.getSenderIndex(); - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("Failure: " + pid.getConnectorDescriptorId() + " sender: " + senderIndex + " receiver: " - + pid.getReceiverIndex()); - } + LOGGER.warn("Failure: " + pid.getConnectorDescriptorId() + " sender: " + senderIndex + " receiver: " + + pid.getReceiverIndex()); failSenders.set(senderIndex); eosSenders.set(senderIndex); notifyAll(); -- To view, visit https://asterix-gerrit.ics.uci.edu/2803 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-MessageType: merged Gerrit-Change-Id: Ia4e24a95aaac37b1d3d0d9a35266109ae0315293 Gerrit-PatchSet: 4 Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Owner: abdullah alamoudi <[email protected]> Gerrit-Reviewer: Anon. E. Moose #1000171 Gerrit-Reviewer: Jenkins <[email protected]> Gerrit-Reviewer: Till Westmann <[email protected]> Gerrit-Reviewer: abdullah alamoudi <[email protected]>
