Michael Blow has uploaded a new change for review.
https://asterix-gerrit.ics.uci.edu/1213
Change subject: Timeout NC thread dump requests after 60 seconds
......................................................................
Timeout NC thread dump requests after 60 seconds
Change-Id: If4840c78a6f6a2916ee682a9061df62a50bedc8a
---
M
hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/GetThreadDumpWork.java
M
hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/NotifyThreadDumpResponse.java
M
hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/work/NodeThreadDumpWork.java
3 files changed, 36 insertions(+), 2 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb
refs/changes/13/1213/1
diff --git
a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/GetThreadDumpWork.java
b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/GetThreadDumpWork.java
index be53232..7931cf8 100644
---
a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/GetThreadDumpWork.java
+++
b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/GetThreadDumpWork.java
@@ -20,6 +20,10 @@
import java.lang.management.ManagementFactory;
import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.logging.Level;
+import java.util.logging.Logger;
import org.apache.hyracks.control.cc.ClusterControllerService;
import org.apache.hyracks.control.cc.NodeControllerState;
@@ -27,6 +31,9 @@
import org.apache.hyracks.control.common.work.ThreadDumpWork;
public class GetThreadDumpWork extends ThreadDumpWork {
+ private static final Logger LOGGER =
Logger.getLogger(ThreadDumpWork.class.getName());
+ public static final int TIMEOUT_SECS = 60;
+
private final ClusterControllerService ccs;
private final String nodeId;
private final IResultCallback<String> callback;
@@ -55,8 +62,27 @@
try {
ncState.getNodeController().takeThreadDump(run.getRequestId());
} catch (Exception e) {
+ ccs.removeThreadDumpRun(run.getRequestId());
callback.setException(e);
}
+ final long requestTime = System.currentTimeMillis();
+ ccs.getExecutor().execute(() -> {
+ try {
+ final long queueTime = System.currentTimeMillis() -
requestTime;
+ final long sleepTime =
TimeUnit.SECONDS.toMillis(TIMEOUT_SECS) - queueTime;
+ if (sleepTime > 0) {
+ Thread.sleep(sleepTime);
+ }
+ if (ccs.removeThreadDumpRun(run.getRequestId()) !=
null) {
+ LOGGER.log(Level.WARNING, "Timed out thread dump
request " + run.getRequestId()
+ + " for node " + nodeId);
+ callback.setException(new TimeoutException("Thread
dump request for node " + nodeId
+ + " timed out after " + TIMEOUT_SECS + "
seconds."));
+ }
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
+ });
}
}
}
diff --git
a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/NotifyThreadDumpResponse.java
b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/NotifyThreadDumpResponse.java
index bbdf211..2dae4b0 100644
---
a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/NotifyThreadDumpResponse.java
+++
b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/NotifyThreadDumpResponse.java
@@ -18,10 +18,13 @@
*/
package org.apache.hyracks.control.cc.work;
+import java.util.logging.Logger;
+
import org.apache.hyracks.control.cc.ClusterControllerService;
import org.apache.hyracks.control.common.work.AbstractWork;
public class NotifyThreadDumpResponse extends AbstractWork {
+ private static final Logger LOGGER =
Logger.getLogger(NotifyThreadDumpResponse.class.getName());
private final ClusterControllerService ccs;
@@ -36,6 +39,12 @@
@Override
public void run() {
-
ccs.removeThreadDumpRun(requestId).notifyThreadDumpReceived(threadDumpJSON);
+ LOGGER.fine("Delivering thread dump response: " + requestId);
+ final GetThreadDumpWork.ThreadDumpRun threadDumpRun =
ccs.removeThreadDumpRun(requestId);
+ if (threadDumpRun == null) {
+ LOGGER.warning("Thread dump run " + requestId + " not found;
discarding reply: " + threadDumpJSON);
+ } else {
+ threadDumpRun.notifyThreadDumpReceived(threadDumpJSON);
+ }
}
}
diff --git
a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/work/NodeThreadDumpWork.java
b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/work/NodeThreadDumpWork.java
index 1fc4690..85233b2 100644
---
a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/work/NodeThreadDumpWork.java
+++
b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/work/NodeThreadDumpWork.java
@@ -33,7 +33,6 @@
@Override
protected void doRun() throws Exception {
final String result = takeDump(ncs.getThreadMXBean());
-
ncs.getClusterController().notifyThreadDump(
ncs.getApplicationContext().getNodeId(), requestId, result);
}
--
To view, visit https://asterix-gerrit.ics.uci.edu/1213
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: If4840c78a6f6a2916ee682a9061df62a50bedc8a
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Michael Blow <[email protected]>
