Michael Blow has submitted this change and it was merged. Change subject: Fix Thread Dump Hang on Newly Dead NC ......................................................................
Fix Thread Dump Hang on Newly Dead NC + improved exception reporting from cluster state HTTP apis Change-Id: I6ed674ec9e6c50c7da62b82694d1807b3338b289 Reviewed-on: https://asterix-gerrit.ics.uci.edu/1207 Sonar-Qube: Jenkins <[email protected]> Tested-by: Jenkins <[email protected]> Integration-Tests: Jenkins <[email protected]> Reviewed-by: Till Westmann <[email protected]> --- M asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/ClusterAPIServlet.java M asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/ClusterCCDetailsAPIServlet.java M asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/ClusterNodeDetailsAPIServlet.java M hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/GetThreadDumpWork.java 4 files changed, 28 insertions(+), 16 deletions(-) Approvals: Till Westmann: Looks good to me, approved Jenkins: Verified; No violations found; Verified diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/ClusterAPIServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/ClusterAPIServlet.java index 64cde54..0d9ff16 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/ClusterAPIServlet.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/ClusterAPIServlet.java @@ -30,7 +30,6 @@ import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; -import org.apache.asterix.app.result.ResultUtil; import org.apache.asterix.common.config.AbstractAsterixProperties; import org.apache.asterix.runtime.util.ClusterStateManager; import org.json.JSONArray; @@ -65,12 +64,11 @@ json = getClusterStateJSON(request, ""); response.setStatus(HttpServletResponse.SC_OK); responseWriter.write(json.toString(4)); - } catch (IllegalArgumentException e) { - ResultUtil.apiErrorHandler(responseWriter, e); - response.setStatus(HttpServletResponse.SC_BAD_REQUEST); + } catch (IllegalArgumentException e) { // NOSONAR - exception not logged or rethrown + response.sendError(HttpServletResponse.SC_BAD_REQUEST); } catch (Exception e) { - ResultUtil.apiErrorHandler(responseWriter, e); - response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR); + LOGGER.log(Level.INFO, "exception thrown for " + request, e); + response.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR, e.toString()); } responseWriter.flush(); } diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/ClusterCCDetailsAPIServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/ClusterCCDetailsAPIServlet.java index 64e65a5..75fba74 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/ClusterCCDetailsAPIServlet.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/ClusterCCDetailsAPIServlet.java @@ -22,17 +22,19 @@ import java.io.IOException; import java.io.PrintWriter; +import java.util.logging.Level; +import java.util.logging.Logger; import javax.servlet.ServletContext; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; -import org.apache.asterix.app.result.ResultUtil; import org.apache.hyracks.api.client.IHyracksClientConnection; import org.json.JSONObject; public class ClusterCCDetailsAPIServlet extends ClusterAPIServlet { private static final long serialVersionUID = 1L; + private static final Logger LOGGER = Logger.getLogger(ClusterCCDetailsAPIServlet.class.getName()); @Override protected void getUnsafe(HttpServletRequest request, HttpServletResponse response) throws IOException { @@ -51,11 +53,11 @@ response.setContentType("application/json"); response.setCharacterEncoding("utf-8"); responseWriter.write(json.toString(4)); - } catch (IllegalArgumentException e) { //NOSONAR - exception not logged or rethrown + } catch (IllegalArgumentException e) { // NOSONAR - exception not logged or rethrown response.sendError(HttpServletResponse.SC_NOT_FOUND); } catch (Exception e) { - ResultUtil.apiErrorHandler(responseWriter, e); - response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR); + LOGGER.log(Level.INFO, "exception thrown for " + request, e); + response.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR, e.toString()); } responseWriter.flush(); } diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/ClusterNodeDetailsAPIServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/ClusterNodeDetailsAPIServlet.java index 5a91fee..f5509c2 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/ClusterNodeDetailsAPIServlet.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/ClusterNodeDetailsAPIServlet.java @@ -25,12 +25,14 @@ import java.util.ArrayList; import java.util.Iterator; import java.util.List; +import java.util.logging.Level; +import java.util.logging.Logger; import javax.servlet.ServletContext; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; -import org.apache.asterix.app.result.ResultUtil; +import org.apache.asterix.runtime.util.ClusterStateManager; import org.apache.hyracks.api.client.IHyracksClientConnection; import org.json.JSONArray; import org.json.JSONException; @@ -38,6 +40,7 @@ public class ClusterNodeDetailsAPIServlet extends ClusterAPIServlet { private static final long serialVersionUID = 1L; + private static final Logger LOGGER = Logger.getLogger(ClusterNodeDetailsAPIServlet.class.getName()); @Override protected void getUnsafe(HttpServletRequest request, HttpServletResponse response) throws IOException { @@ -57,11 +60,13 @@ response.setContentType("application/json"); response.setCharacterEncoding("utf-8"); responseWriter.write(json.toString(4)); - } catch (IllegalArgumentException e) { //NOSONAR - exception not logged or rethrown + } catch (IllegalStateException e) { // NOSONAR - exception not logged or rethrown + response.sendError(HttpServletResponse.SC_SERVICE_UNAVAILABLE); + } catch (IllegalArgumentException e) { // NOSONAR - exception not logged or rethrown response.sendError(HttpServletResponse.SC_NOT_FOUND); } catch (Exception e) { - ResultUtil.apiErrorHandler(responseWriter, e); - response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR); + LOGGER.log(Level.INFO, "exception thrown for " + request, e); + response.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR, e.toString()); } responseWriter.flush(); } @@ -187,7 +192,10 @@ } String dump = hcc.getThreadDump(node); if (dump == null) { - throw new IllegalArgumentException(); + // check to see if this is a node that is simply down + throw ClusterStateManager.INSTANCE.getNodePartitions(node) != null + ? new IllegalStateException() + : new IllegalArgumentException(); } return new JSONObject(dump); } diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/GetThreadDumpWork.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/GetThreadDumpWork.java index b18d388..be53232 100644 --- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/GetThreadDumpWork.java +++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/GetThreadDumpWork.java @@ -52,7 +52,11 @@ callback.setValue(null); } else { ccs.addThreadDumpRun(run.getRequestId(), run); - ncState.getNodeController().takeThreadDump(run.getRequestId()); + try { + ncState.getNodeController().takeThreadDump(run.getRequestId()); + } catch (Exception e) { + callback.setException(e); + } } } } -- To view, visit https://asterix-gerrit.ics.uci.edu/1207 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-MessageType: merged Gerrit-Change-Id: I6ed674ec9e6c50c7da62b82694d1807b3338b289 Gerrit-PatchSet: 2 Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Owner: Michael Blow <[email protected]> Gerrit-Reviewer: Jenkins <[email protected]> Gerrit-Reviewer: Michael Blow <[email protected]> Gerrit-Reviewer: Till Westmann <[email protected]>
