Repository: asterixdb
Updated Branches:
  refs/heads/master bb49e5e39 -> 12009bd40


[NO ISSUE] Refactor shutdown watchdog to ExitUtil

Change-Id: Id27635f90587c64ee29ab1a4ac2b1a6182042900
Reviewed-on: https://asterix-gerrit.ics.uci.edu/2470
Reviewed-by: Murtadha Hubail <mhub...@apache.org>
Sonar-Qube: Jenkins <jenk...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <jenk...@fulliautomatix.ics.uci.edu>
Contrib: Jenkins <jenk...@fulliautomatix.ics.uci.edu>
Integration-Tests: Jenkins <jenk...@fulliautomatix.ics.uci.edu>


Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/12009bd4
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/12009bd4
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/12009bd4

Branch: refs/heads/master
Commit: 12009bd40701d44b206e4c65fe0561279f554235
Parents: bb49e5e
Author: Michael Blow <mb...@apache.org>
Authored: Sat Mar 10 14:09:44 2018 -0800
Committer: Michael Blow <mb...@apache.org>
Committed: Sat Mar 10 23:26:20 2018 -0800

----------------------------------------------------------------------
 .../RegistrationTasksResponseMessage.java       |  3 +-
 .../bootstrap/GlobalRecoveryManager.java        |  3 +-
 .../management/service/logging/LogManager.java  |  4 +-
 .../lifecycle/LifeCycleComponentManager.java    |  4 +-
 .../control/cc/work/ClusterShutdownWork.java    |  5 +-
 .../hyracks/control/nc/NCShutdownHook.java      | 28 +--------
 .../hyracks/control/nc/task/ShutdownTask.java   |  5 +-
 .../java/org/apache/hyracks/util/ExitUtil.java  | 63 ++++++++++++++++++--
 8 files changed, 76 insertions(+), 39 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/asterixdb/blob/12009bd4/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/message/RegistrationTasksResponseMessage.java
----------------------------------------------------------------------
diff --git 
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/message/RegistrationTasksResponseMessage.java
 
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/message/RegistrationTasksResponseMessage.java
index a6f10ca..868c2ad 100644
--- 
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/message/RegistrationTasksResponseMessage.java
+++ 
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/message/RegistrationTasksResponseMessage.java
@@ -28,7 +28,6 @@ import 
org.apache.asterix.common.messaging.api.INcAddressedMessage;
 import org.apache.asterix.common.replication.INCLifecycleMessage;
 import org.apache.hyracks.api.exceptions.HyracksDataException;
 import org.apache.hyracks.api.service.IControllerService;
-import org.apache.hyracks.control.nc.NCShutdownHook;
 import org.apache.hyracks.util.ExitUtil;
 import org.apache.logging.log4j.Level;
 import org.apache.logging.log4j.LogManager;
@@ -80,7 +79,7 @@ public class RegistrationTasksResponseMessage extends 
CcIdentifiedMessage
         } finally {
             if (!success) {
                 // stop NC so that it can be started again
-                ExitUtil.exit(NCShutdownHook.FAILED_TO_STARTUP_EXIT_CODE);
+                ExitUtil.exit(ExitUtil.EC_FAILED_TO_STARTUP);
             }
         }
     }

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/12009bd4/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/GlobalRecoveryManager.java
----------------------------------------------------------------------
diff --git 
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/GlobalRecoveryManager.java
 
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/GlobalRecoveryManager.java
index e2f1eaf..3d9b822 100644
--- 
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/GlobalRecoveryManager.java
+++ 
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/GlobalRecoveryManager.java
@@ -46,7 +46,6 @@ import org.apache.hyracks.api.client.IHyracksClientConnection;
 import org.apache.hyracks.api.exceptions.HyracksDataException;
 import org.apache.hyracks.api.job.JobId;
 import org.apache.hyracks.api.job.JobSpecification;
-import org.apache.hyracks.control.nc.NCShutdownHook;
 import org.apache.hyracks.util.ExitUtil;
 import org.apache.logging.log4j.Level;
 import org.apache.logging.log4j.LogManager;
@@ -99,7 +98,7 @@ public class GlobalRecoveryManager implements 
IGlobalRecoveryManager {
                             recover(appCtx);
                         } catch (HyracksDataException e) {
                             LOGGER.log(Level.ERROR, "Global recovery failed. 
Shutting down...", e);
-                            
ExitUtil.exit(NCShutdownHook.FAILED_TO_RECOVER_EXIT_CODE);
+                            ExitUtil.exit(ExitUtil.EC_FAILED_TO_RECOVER);
                         }
                     });
                 }

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/12009bd4/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/service/logging/LogManager.java
----------------------------------------------------------------------
diff --git 
a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/service/logging/LogManager.java
 
b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/service/logging/LogManager.java
index ce65de0..6208cef 100644
--- 
a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/service/logging/LogManager.java
+++ 
b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/service/logging/LogManager.java
@@ -18,6 +18,8 @@
  */
 package org.apache.asterix.transaction.management.service.logging;
 
+import static org.apache.hyracks.util.ExitUtil.EC_IMMEDIATE_HALT;
+
 import java.io.File;
 import java.io.FilenameFilter;
 import java.io.IOException;
@@ -716,7 +718,7 @@ class LogFlusher implements Callable<Boolean> {
             }
         } catch (Exception e) {
             LOGGER.log(Level.ERROR, "LogFlusher is terminating abnormally. 
System is in unusable state; halting", e);
-            ExitUtil.halt(44);
+            ExitUtil.halt(EC_IMMEDIATE_HALT);
             throw new AssertionError("not reachable");
         } finally {
             if (interrupted) {

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/12009bd4/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/lifecycle/LifeCycleComponentManager.java
----------------------------------------------------------------------
diff --git 
a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/lifecycle/LifeCycleComponentManager.java
 
b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/lifecycle/LifeCycleComponentManager.java
index 6d5d246..a9b7a97 100644
--- 
a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/lifecycle/LifeCycleComponentManager.java
+++ 
b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/lifecycle/LifeCycleComponentManager.java
@@ -18,6 +18,8 @@
  */
 package org.apache.hyracks.api.lifecycle;
 
+import static org.apache.hyracks.util.ExitUtil.EC_UNHANDLED_EXCEPTION;
+
 import java.io.File;
 import java.io.FileOutputStream;
 import java.io.IOException;
@@ -57,7 +59,7 @@ public class LifeCycleComponentManager implements 
ILifeCycleComponentManager {
         try {
             LOGGER.log(Level.ERROR, "Uncaught Exception from thread " + 
t.getName() + ". Calling shutdown hook", e);
         } finally {
-            ExitUtil.exit(99);
+            ExitUtil.exit(EC_UNHANDLED_EXCEPTION);
         }
     }
 

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/12009bd4/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/ClusterShutdownWork.java
----------------------------------------------------------------------
diff --git 
a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/ClusterShutdownWork.java
 
b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/ClusterShutdownWork.java
index a7c3c2f..194d27f 100644
--- 
a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/ClusterShutdownWork.java
+++ 
b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/ClusterShutdownWork.java
@@ -19,6 +19,9 @@
 
 package org.apache.hyracks.control.cc.work;
 
+import static org.apache.hyracks.util.ExitUtil.EC_ABNORMAL_TERMINATION;
+import static org.apache.hyracks.util.ExitUtil.EC_NORMAL_TERMINATION;
+
 import java.util.Collection;
 
 import org.apache.hyracks.control.cc.ClusterControllerService;
@@ -82,7 +85,7 @@ public class ClusterShutdownWork extends SynchronizableWork {
                     }
                     callback.setValue(cleanShutdown);
                     ccs.stop(terminateNCService);
-                    ExitUtil.exit(cleanShutdown ? 0 : 1);
+                    ExitUtil.exit(cleanShutdown ? EC_NORMAL_TERMINATION : 
EC_ABNORMAL_TERMINATION);
                 } catch (Exception e) {
                     callback.setException(e);
                 }

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/12009bd4/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NCShutdownHook.java
----------------------------------------------------------------------
diff --git 
a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NCShutdownHook.java
 
b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NCShutdownHook.java
index ccac00e..9a19f8e 100644
--- 
a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NCShutdownHook.java
+++ 
b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NCShutdownHook.java
@@ -18,7 +18,6 @@
  */
 package org.apache.hyracks.control.nc;
 
-import org.apache.hyracks.util.ExitUtil;
 import org.apache.hyracks.util.ThreadDumpUtil;
 import org.apache.logging.log4j.Level;
 import org.apache.logging.log4j.LogManager;
@@ -31,35 +30,12 @@ import org.apache.logging.log4j.Logger;
  */
 public class NCShutdownHook extends Thread {
 
-    public static final int FAILED_TO_STARTUP_EXIT_CODE = 2;
-    public static final int FAILED_TO_RECOVER_EXIT_CODE = 3;
     private static final Logger LOGGER = LogManager.getLogger();
-    private static final long SHUTDOWN_WAIT_TIME = 10 * 60 * 1000L;
-    private final Thread watchDog;
     private final NodeControllerService nodeControllerService;
-    private volatile Thread shutdownHookThread;
 
-    public NCShutdownHook(NodeControllerService nodeControllerService) {
+    NCShutdownHook(NodeControllerService nodeControllerService) {
         super("ShutdownHook-" + nodeControllerService.getId());
         this.nodeControllerService = nodeControllerService;
-        watchDog = new Thread(watch(), "ShutdownHookWatchDog-" + 
nodeControllerService.getId());
-    }
-
-    private Runnable watch() {
-        return () -> {
-            try {
-                shutdownHookThread.join(SHUTDOWN_WAIT_TIME); // 10 min
-                if (shutdownHookThread.isAlive()) {
-                    try {
-                        LOGGER.info("Watchdog is angry. Killing shutdown 
hook");
-                    } finally {
-                        ExitUtil.halt(ExitUtil.EXIT_CODE_SHUTDOWN_TIMED_OUT);
-                    }
-                }
-            } catch (Throwable th) { // NOSONAR must catch them all
-                ExitUtil.halt(ExitUtil.EXIT_CODE_WATCHDOG_FAILED);
-            }
-        };
     }
 
     @Override
@@ -69,8 +45,6 @@ public class NCShutdownHook extends Thread {
                 LOGGER.info("Shutdown hook called");
             } catch (Throwable th) {//NOSONAR
             }
-            shutdownHookThread = Thread.currentThread();
-            watchDog.start();
             LOGGER.log(Level.INFO, () -> "Thread dump at shutdown: " + 
ThreadDumpUtil.takeDumpString());
             nodeControllerService.stop();
         } catch (Throwable th) { // NOSONAR... This is fine since this is 
shutdown hook

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/12009bd4/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/task/ShutdownTask.java
----------------------------------------------------------------------
diff --git 
a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/task/ShutdownTask.java
 
b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/task/ShutdownTask.java
index 4dd57f2..ba8c3da 100644
--- 
a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/task/ShutdownTask.java
+++ 
b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/task/ShutdownTask.java
@@ -19,6 +19,9 @@
 
 package org.apache.hyracks.control.nc.task;
 
+import static org.apache.hyracks.util.ExitUtil.EC_NORMAL_TERMINATION;
+import static 
org.apache.hyracks.util.ExitUtil.EC_TERMINATE_NC_SERVICE_DIRECTIVE;
+
 import org.apache.hyracks.util.ExitUtil;
 
 public class ShutdownTask implements Runnable {
@@ -30,7 +33,7 @@ public class ShutdownTask implements Runnable {
 
     @Override
     public void run() {
-        ExitUtil.exit(terminateNCService ? 99 : 0);
+        ExitUtil.exit(terminateNCService ? EC_TERMINATE_NC_SERVICE_DIRECTIVE : 
EC_NORMAL_TERMINATION);
     }
 
 }

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/12009bd4/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/ExitUtil.java
----------------------------------------------------------------------
diff --git 
a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/ExitUtil.java
 
b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/ExitUtil.java
index 523480a..ec6587e 100644
--- 
a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/ExitUtil.java
+++ 
b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/ExitUtil.java
@@ -18,6 +18,9 @@
  */
 package org.apache.hyracks.util;
 
+import java.util.concurrent.TimeUnit;
+
+import org.apache.commons.lang3.mutable.MutableLong;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
@@ -26,10 +29,26 @@ public class ExitUtil {
 
     private static final Logger LOGGER = LogManager.getLogger();
 
+    public static final int EC_NORMAL_TERMINATION = 0;
+    public static final int EC_ABNORMAL_TERMINATION = 1;
+    public static final int EC_FAILED_TO_STARTUP = 2;
+    public static final int EC_FAILED_TO_RECOVER = 3;
+    public static final int EC_UNHANDLED_EXCEPTION = 11;
+    public static final int EC_IMMEDIATE_HALT = 33;
+    public static final int EC_HALT_ABNORMAL_RESERVED_44 = 44;
+    public static final int EC_HALT_ABNORMAL_RESERVED_55 = 55;
+    public static final int EC_HALT_SHUTDOWN_TIMED_OUT = 66;
+    public static final int EC_HALT_WATCHDOG_FAILED = 77;
+    public static final int EC_HALT_ABNORMAL_RESERVED_88 = 88;
+    public static final int EC_TERMINATE_NC_SERVICE_DIRECTIVE = 99;
+
     private static final ExitThread exitThread = new ExitThread();
+    private static final ShutdownWatchdog watchdogThread = new 
ShutdownWatchdog();
+    private static final MutableLong shutdownHaltDelay = new MutableLong(10 * 
60 * 1000L); // 10 minutes default
 
-    public static final int EXIT_CODE_SHUTDOWN_TIMED_OUT = 66;
-    public static final int EXIT_CODE_WATCHDOG_FAILED = 77;
+    static {
+        Runtime.getRuntime().addShutdownHook(new 
Thread(watchdogThread::start));
+    }
 
     private ExitUtil() {
     }
@@ -39,8 +58,20 @@ public class ExitUtil {
     }
 
     public static void exit(int status) {
-        exitThread.setStatus(status);
-        exitThread.start();
+        synchronized (exitThread) {
+            if (exitThread.isAlive()) {
+                LOGGER.warn("ignoring duplicate request to exit with status " 
+ status
+                        + "; already exiting with status " + exitThread.status 
+ "...");
+            } else {
+                exitThread.setStatus(status);
+                exitThread.start();
+            }
+        }
+    }
+
+    public static void exit(int status, long timeBeforeHalt, TimeUnit 
timeBeforeHaltUnit) {
+        
shutdownHaltDelay.setValue(timeBeforeHaltUnit.toMillis(timeBeforeHalt));
+        exit(status);
     }
 
     @SuppressWarnings("squid:S2142") // catch interrupted
@@ -55,6 +86,30 @@ public class ExitUtil {
         Runtime.getRuntime().halt(status);
     }
 
+    private static class ShutdownWatchdog extends Thread {
+
+        private ShutdownWatchdog() {
+            super("ShutdownWatchdog");
+            setDaemon(true);
+        }
+
+        @Override
+        public void run() {
+            try {
+                exitThread.join(shutdownHaltDelay.getValue()); // 10 min
+                if (exitThread.isAlive()) {
+                    try {
+                        LOGGER.info("Watchdog is angry. Killing shutdown 
hook");
+                    } finally {
+                        ExitUtil.halt(EC_HALT_SHUTDOWN_TIMED_OUT);
+                    }
+                }
+            } catch (Throwable th) { // NOSONAR must catch them all
+                ExitUtil.halt(EC_HALT_WATCHDOG_FAILED);
+            }
+        }
+    }
+
     private static class ExitThread extends Thread {
         private int status;
 

Reply via email to