Change in asterixdb[master]: [NO ISSUE][TX][FAIL] Halt on LogFlusher failure

[Michael Blow (Code Review)]

Michael Blow has uploaded a new change for review.

  https://asterix-gerrit.ics.uci.edu/2464

Change subject: [NO ISSUE][TX][FAIL] Halt on LogFlusher failure
......................................................................

[NO ISSUE][TX][FAIL] Halt on LogFlusher failure

Change-Id: I69c4bf6c9669601e6613a9435de046d72b0baf6d
---
M 
asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/service/logging/LogManager.java
M 
hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NCShutdownHook.java
M 
hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/ExitUtil.java
3 files changed, 22 insertions(+), 4 deletions(-)


  git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb 
refs/changes/64/2464/1

diff --git 
a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/service/logging/LogManager.java
 
b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/service/logging/LogManager.java
index e3788b7..ce65de0 100644
--- 
a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/service/logging/LogManager.java
+++ 
b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/service/logging/LogManager.java
@@ -56,6 +56,7 @@
 import org.apache.asterix.common.transactions.TxnLogFile;
 import org.apache.hyracks.api.lifecycle.ILifeCycleComponent;
 import org.apache.hyracks.api.util.InvokeUtil;
+import org.apache.hyracks.util.ExitUtil;
 import org.apache.logging.log4j.Level;
 import org.apache.logging.log4j.Logger;
 
@@ -714,8 +715,9 @@
                 emptyQ.add(flushPage.getLogPageSize() == 
logMgr.getLogPageSize() ? flushPage : stashQ.remove());
             }
         } catch (Exception e) {
-            LOGGER.log(Level.ERROR, "LogFlusher is terminating abnormally. 
System is in unusable state.", e);
-            throw e;
+            LOGGER.log(Level.ERROR, "LogFlusher is terminating abnormally. 
System is in unusable state; halting", e);
+            ExitUtil.halt(44);
+            throw new AssertionError("not reachable");
         } finally {
             if (interrupted) {
                 Thread.currentThread().interrupt();
diff --git 
a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NCShutdownHook.java
 
b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NCShutdownHook.java
index 020e564..ccac00e 100644
--- 
a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NCShutdownHook.java
+++ 
b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NCShutdownHook.java
@@ -18,6 +18,7 @@
  */
 package org.apache.hyracks.control.nc;
 
+import org.apache.hyracks.util.ExitUtil;
 import org.apache.hyracks.util.ThreadDumpUtil;
 import org.apache.logging.log4j.Level;
 import org.apache.logging.log4j.LogManager;
@@ -52,11 +53,11 @@
                     try {
                         LOGGER.info("Watchdog is angry. Killing shutdown 
hook");
                     } finally {
-                        Runtime.getRuntime().halt(66); // NOSONAR last resort
+                        ExitUtil.halt(ExitUtil.EXIT_CODE_SHUTDOWN_TIMED_OUT);
                     }
                 }
             } catch (Throwable th) { // NOSONAR must catch them all
-                Runtime.getRuntime().halt(77); // NOSONAR last resort
+                ExitUtil.halt(ExitUtil.EXIT_CODE_WATCHDOG_FAILED);
             }
         };
     }
diff --git 
a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/ExitUtil.java
 
b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/ExitUtil.java
index c54c9dc..523480a 100644
--- 
a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/ExitUtil.java
+++ 
b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/ExitUtil.java
@@ -28,6 +28,9 @@
 
     private static final ExitThread exitThread = new ExitThread();
 
+    public static final int EXIT_CODE_SHUTDOWN_TIMED_OUT = 66;
+    public static final int EXIT_CODE_WATCHDOG_FAILED = 77;
+
     private ExitUtil() {
     }
 
@@ -40,6 +43,18 @@
         exitThread.start();
     }
 
+    @SuppressWarnings("squid:S2142") // catch interrupted
+    public static void halt(int status) {
+        LOGGER.fatal("JVM halting with status " + status + "; bye!", new 
Throwable("halt stacktrace"));
+        try {
+            // try to give time for the log to be emitted...
+            Thread.sleep(2000);
+        } catch (InterruptedException e) {
+            // ignore
+        }
+        Runtime.getRuntime().halt(status);
+    }
+
     private static class ExitThread extends Thread {
         private int status;
 

-- 
To view, visit https://asterix-gerrit.ics.uci.edu/2464
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: I69c4bf6c9669601e6613a9435de046d72b0baf6d
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Michael Blow <mb...@apache.org>