Michael Blow has submitted this change and it was merged. Change subject: [NO ISSUE] Ensure shutdown watchdog is started ......................................................................
[NO ISSUE] Ensure shutdown watchdog is started Relying on the shutdown watchdog to be triggered by a shutdown hook is problematic, as it is undefined when the JVM will run the shutdown hook. Change-Id: Ia3ed2c88ec4c93a298f89f12f29448370cae9136 Reviewed-on: https://asterix-gerrit.ics.uci.edu/2860 Reviewed-by: Michael Blow <[email protected]> Tested-by: Michael Blow <[email protected]> --- M hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/ExitUtil.java 1 file changed, 29 insertions(+), 3 deletions(-) Approvals: Michael Blow: Looks good to me, approved; Verified diff --git a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/ExitUtil.java b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/ExitUtil.java index 6aa708d..9604c30 100644 --- a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/ExitUtil.java +++ b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/ExitUtil.java @@ -18,6 +18,9 @@ */ package org.apache.hyracks.util; +import java.lang.reflect.Field; +import java.util.IdentityHashMap; +import java.util.concurrent.Semaphore; import java.util.concurrent.TimeUnit; import org.apache.commons.lang3.mutable.MutableLong; @@ -58,7 +61,7 @@ private static final MutableLong shutdownHaltDelay = new MutableLong(10 * 60 * 1000L); // 10 minutes default static { - Runtime.getRuntime().addShutdownHook(new Thread(watchdogThread::start)); + watchdogThread.start(); } private ExitUtil() { @@ -94,6 +97,8 @@ private static class ShutdownWatchdog extends Thread { + private final Semaphore startSemaphore = new Semaphore(0); + private ShutdownWatchdog() { super("ShutdownWatchdog"); setDaemon(true); @@ -101,11 +106,14 @@ @Override public void run() { + startSemaphore.acquireUninterruptibly(); + LOGGER.info("starting shutdown watchdog- system will halt if shutdown is not completed within {} seconds", + TimeUnit.MILLISECONDS.toSeconds(shutdownHaltDelay.getValue())); try { - exitThread.join(shutdownHaltDelay.getValue()); // 10 min + exitThread.join(shutdownHaltDelay.getValue()); if (exitThread.isAlive()) { try { - LOGGER.warn("Watchdog is angry. Killing shutdown hook"); + LOGGER.fatal("shutdown did not complete within configured delay; halting"); } finally { ExitUtil.halt(EC_HALT_SHUTDOWN_TIMED_OUT); } @@ -113,6 +121,10 @@ } catch (Throwable th) { // NOSONAR must catch them all ExitUtil.halt(EC_HALT_WATCHDOG_FAILED); } + } + + public void beginWatch() { + startSemaphore.release(); } } @@ -127,8 +139,10 @@ @Override public void run() { + watchdogThread.beginWatch(); try { LOGGER.warn("JVM exiting with status " + status + "; bye!", callstack); + logShutdownHooks(); } finally { Runtime.getRuntime().exit(status); } @@ -138,5 +152,17 @@ this.status = status; this.callstack = callstack; } + + private static void logShutdownHooks() { + try { + Class clazz = Class.forName("java.lang.ApplicationShutdownHooks"); + Field hooksField = clazz.getDeclaredField("hooks"); + hooksField.setAccessible(true); + IdentityHashMap hooks = (IdentityHashMap) hooksField.get(null); + LOGGER.info("the following ({}) shutdown hooks have been registered: {}", hooks::size, hooks::toString); + } catch (Exception e) { + LOGGER.warn("ignoring exception trying to determine number of shutdown hooks", e); + } + } } } -- To view, visit https://asterix-gerrit.ics.uci.edu/2860 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-MessageType: merged Gerrit-Change-Id: Ia3ed2c88ec4c93a298f89f12f29448370cae9136 Gerrit-PatchSet: 4 Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Owner: Michael Blow <[email protected]> Gerrit-Reviewer: Anon. E. Moose #1000171 Gerrit-Reviewer: Jenkins <[email protected]> Gerrit-Reviewer: Michael Blow <[email protected]> Gerrit-Reviewer: Murtadha Hubail <[email protected]> Gerrit-Reviewer: Till Westmann <[email protected]>
