Michael Blow has submitted this change and it was merged. Change subject: Timeout NC thread dump requests after 60 seconds ......................................................................
Timeout NC thread dump requests after 60 seconds Change-Id: If4840c78a6f6a2916ee682a9061df62a50bedc8a Reviewed-on: https://asterix-gerrit.ics.uci.edu/1213 Sonar-Qube: Jenkins <[email protected]> Tested-by: Jenkins <[email protected]> Integration-Tests: Jenkins <[email protected]> Reviewed-by: Till Westmann <[email protected]> --- M hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/GetThreadDumpWork.java M hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/NotifyThreadDumpResponse.java M hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/work/NodeThreadDumpWork.java 3 files changed, 36 insertions(+), 2 deletions(-) Approvals: Till Westmann: Looks good to me, approved Jenkins: Verified; No violations found; Verified diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/GetThreadDumpWork.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/GetThreadDumpWork.java index be53232..7931cf8 100644 --- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/GetThreadDumpWork.java +++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/GetThreadDumpWork.java @@ -20,6 +20,10 @@ import java.lang.management.ManagementFactory; import java.util.UUID; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.logging.Level; +import java.util.logging.Logger; import org.apache.hyracks.control.cc.ClusterControllerService; import org.apache.hyracks.control.cc.NodeControllerState; @@ -27,6 +31,9 @@ import org.apache.hyracks.control.common.work.ThreadDumpWork; public class GetThreadDumpWork extends ThreadDumpWork { + private static final Logger LOGGER = Logger.getLogger(ThreadDumpWork.class.getName()); + public static final int TIMEOUT_SECS = 60; + private final ClusterControllerService ccs; private final String nodeId; private final IResultCallback<String> callback; @@ -55,8 +62,27 @@ try { ncState.getNodeController().takeThreadDump(run.getRequestId()); } catch (Exception e) { + ccs.removeThreadDumpRun(run.getRequestId()); callback.setException(e); } + final long requestTime = System.currentTimeMillis(); + ccs.getExecutor().execute(() -> { + try { + final long queueTime = System.currentTimeMillis() - requestTime; + final long sleepTime = TimeUnit.SECONDS.toMillis(TIMEOUT_SECS) - queueTime; + if (sleepTime > 0) { + Thread.sleep(sleepTime); + } + if (ccs.removeThreadDumpRun(run.getRequestId()) != null) { + LOGGER.log(Level.WARNING, "Timed out thread dump request " + run.getRequestId() + + " for node " + nodeId); + callback.setException(new TimeoutException("Thread dump request for node " + nodeId + + " timed out after " + TIMEOUT_SECS + " seconds.")); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + }); } } } diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/NotifyThreadDumpResponse.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/NotifyThreadDumpResponse.java index bbdf211..2dae4b0 100644 --- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/NotifyThreadDumpResponse.java +++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/NotifyThreadDumpResponse.java @@ -18,10 +18,13 @@ */ package org.apache.hyracks.control.cc.work; +import java.util.logging.Logger; + import org.apache.hyracks.control.cc.ClusterControllerService; import org.apache.hyracks.control.common.work.AbstractWork; public class NotifyThreadDumpResponse extends AbstractWork { + private static final Logger LOGGER = Logger.getLogger(NotifyThreadDumpResponse.class.getName()); private final ClusterControllerService ccs; @@ -36,6 +39,12 @@ @Override public void run() { - ccs.removeThreadDumpRun(requestId).notifyThreadDumpReceived(threadDumpJSON); + LOGGER.fine("Delivering thread dump response: " + requestId); + final GetThreadDumpWork.ThreadDumpRun threadDumpRun = ccs.removeThreadDumpRun(requestId); + if (threadDumpRun == null) { + LOGGER.warning("Thread dump run " + requestId + " not found; discarding reply: " + threadDumpJSON); + } else { + threadDumpRun.notifyThreadDumpReceived(threadDumpJSON); + } } } diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/work/NodeThreadDumpWork.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/work/NodeThreadDumpWork.java index 1fc4690..85233b2 100644 --- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/work/NodeThreadDumpWork.java +++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/work/NodeThreadDumpWork.java @@ -33,7 +33,6 @@ @Override protected void doRun() throws Exception { final String result = takeDump(ncs.getThreadMXBean()); - ncs.getClusterController().notifyThreadDump( ncs.getApplicationContext().getNodeId(), requestId, result); } -- To view, visit https://asterix-gerrit.ics.uci.edu/1213 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-MessageType: merged Gerrit-Change-Id: If4840c78a6f6a2916ee682a9061df62a50bedc8a Gerrit-PatchSet: 2 Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Owner: Michael Blow <[email protected]> Gerrit-Reviewer: Jenkins <[email protected]> Gerrit-Reviewer: Michael Blow <[email protected]> Gerrit-Reviewer: Till Westmann <[email protected]>
