Michael Blow has submitted this change and it was merged. Change subject: [NO ISSUE][HYR][*DB] Exit JVM from independent thread to avoid deadlocks ......................................................................
[NO ISSUE][HYR][*DB] Exit JVM from independent thread to avoid deadlocks Change-Id: I21b2090ea3ef85e95ae90de04b08b4a6d22ebe42 Reviewed-on: https://asterix-gerrit.ics.uci.edu/1973 Integration-Tests: Jenkins <[email protected]> Tested-by: Jenkins <[email protected]> Reviewed-by: Murtadha Hubail <[email protected]> --- M asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/message/StartupTaskResponseMessage.java M hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/lifecycle/LifeCycleComponentManager.java M hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/pom.xml M hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/ClusterControllerService.java M hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/ClusterShutdownWork.java M hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/pom.xml M hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NodeControllerService.java M hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/task/ShutdownTask.java A hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/ExitUtil.java 9 files changed, 109 insertions(+), 34 deletions(-) Approvals: Anon. E. Moose #1000171: Jenkins: Verified; Verified Murtadha Hubail: Looks good to me, approved Objections: Jenkins: Violations found; diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/message/StartupTaskResponseMessage.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/message/StartupTaskResponseMessage.java index 9125857..aaf8918 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/message/StartupTaskResponseMessage.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/message/StartupTaskResponseMessage.java @@ -30,6 +30,7 @@ import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.api.service.IControllerService; import org.apache.hyracks.control.nc.NCShutdownHook; +import org.apache.hyracks.util.ExitUtil; public class StartupTaskResponseMessage implements INCLifecycleMessage, INcAddressedMessage { @@ -69,7 +70,7 @@ } finally { if (!success) { // stop NC so that it can be started again - Runtime.getRuntime().exit(NCShutdownHook.FAILED_TO_STARTUP_EXIT_CODE); //NOSONAR startup failed + ExitUtil.exit(NCShutdownHook.FAILED_TO_STARTUP_EXIT_CODE); } } } diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/lifecycle/LifeCycleComponentManager.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/lifecycle/LifeCycleComponentManager.java index 76fa322..f5b4417 100644 --- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/lifecycle/LifeCycleComponentManager.java +++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/lifecycle/LifeCycleComponentManager.java @@ -28,6 +28,8 @@ import java.util.logging.Level; import java.util.logging.Logger; +import org.apache.hyracks.util.ExitUtil; + public class LifeCycleComponentManager implements ILifeCycleComponentManager { public static final class Config { @@ -54,7 +56,7 @@ try { LOGGER.log(Level.SEVERE, "Uncaught Exception from thread " + t.getName() + ". Calling shutdown hook", e); } finally { - Runtime.getRuntime().exit(99);// NOSONAR: It is really required + ExitUtil.exit(99); } } diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/pom.xml b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/pom.xml index b28cc79..fce37dd 100644 --- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/pom.xml +++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/pom.xml @@ -43,6 +43,11 @@ <version>${project.version}</version> </dependency> <dependency> + <groupId>org.apache.hyracks</groupId> + <artifactId>hyracks-util</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> <groupId>io.netty</groupId> <artifactId>netty-all</artifactId> </dependency> diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/ClusterControllerService.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/ClusterControllerService.java index d32e577..a243bf8 100644 --- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/ClusterControllerService.java +++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/ClusterControllerService.java @@ -83,6 +83,7 @@ import org.apache.hyracks.ipc.api.IIPCI; import org.apache.hyracks.ipc.impl.IPCSystem; import org.apache.hyracks.ipc.impl.JavaSerializationBasedPayloadSerializerDeserializer; +import org.apache.hyracks.util.ExitUtil; import org.xml.sax.InputSource; public class ClusterControllerService implements IControllerService { @@ -138,6 +139,10 @@ private ShutdownRun shutdownCallback; + static { + ExitUtil.init(); + } + public ClusterControllerService(final CCConfig config) throws Exception { this(config, getApplication(config)); } diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/ClusterShutdownWork.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/ClusterShutdownWork.java index 0b89f55..613efad 100644 --- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/ClusterShutdownWork.java +++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/ClusterShutdownWork.java @@ -30,6 +30,7 @@ import org.apache.hyracks.control.common.work.IResultCallback; import org.apache.hyracks.control.common.work.SynchronizableWork; import org.apache.hyracks.ipc.exceptions.IPCException; +import org.apache.hyracks.util.ExitUtil; public class ClusterShutdownWork extends SynchronizableWork { private static final Logger LOGGER = Logger.getLogger(ClusterShutdownWork.class.getName()); @@ -53,41 +54,36 @@ } INodeManager nodeManager = ccs.getNodeManager(); Collection<String> nodeIds = nodeManager.getAllNodeIds(); - /** + /* * set up our listener for the node ACKs */ final ShutdownRun shutdownStatus = new ShutdownRun(nodeIds); // set up the CC to listen for it ccs.setShutdownRun(shutdownStatus); - /** + /* * Shutdown all the nodes... */ nodeManager.apply(this::shutdownNode); - ccs.getExecutor().execute(new Runnable() { - @Override - public void run() { - try { + ccs.getExecutor().execute(() -> { + try { + /* + * wait for all our acks + */ + LOGGER.info("Waiting for NCs to shutdown..."); + boolean cleanShutdown = shutdownStatus.waitForCompletion(); + if (!cleanShutdown) { /* - * wait for all our acks + * best effort - just exit, user will have to kill misbehaving NCs */ - LOGGER.info("Waiting for NCs to shutdown..."); - boolean cleanShutdown = shutdownStatus.waitForCompletion(); - if (!cleanShutdown) { - /* - * best effort - just exit, user will have to kill misbehaving NCs - */ - LOGGER.severe("Clean shutdown of NCs timed out- giving up; unresponsive nodes: " + - shutdownStatus.getRemainingNodes()); - } - callback.setValue(cleanShutdown); - ccs.stop(terminateNCService); - LOGGER.info("JVM Exiting.. Bye!"); - Runtime rt = Runtime.getRuntime(); - rt.exit(cleanShutdown ? 0 : 1); - } catch (Exception e) { - callback.setException(e); + LOGGER.severe("Clean shutdown of NCs timed out- giving up; unresponsive nodes: " + + shutdownStatus.getRemainingNodes()); } + callback.setValue(cleanShutdown); + ccs.stop(terminateNCService); + ExitUtil.exit(cleanShutdown ? 0 : 1); + } catch (Exception e) { + callback.setException(e); } }); } catch (Exception e) { diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/pom.xml b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/pom.xml index 548ed31..b5e96e3 100644 --- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/pom.xml +++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/pom.xml @@ -54,6 +54,11 @@ <version>${project.version}</version> </dependency> <dependency> + <groupId>org.apache.hyracks</groupId> + <artifactId>hyracks-util</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> <groupId>commons-io</groupId> <artifactId>commons-io</artifactId> </dependency> diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NodeControllerService.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NodeControllerService.java index 5601f9c..b52675c 100644 --- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NodeControllerService.java +++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NodeControllerService.java @@ -89,6 +89,7 @@ import org.apache.hyracks.ipc.impl.IPCSystem; import org.apache.hyracks.net.protocols.muxdemux.FullFrameChannelInterfaceFactory; import org.apache.hyracks.net.protocols.muxdemux.MuxDemuxPerformanceCounters; +import org.apache.hyracks.util.ExitUtil; import org.kohsuke.args4j.CmdLineException; public class NodeControllerService implements IControllerService { @@ -166,6 +167,10 @@ private final AtomicLong maxJobId = new AtomicLong(-1); + static { + ExitUtil.init(); + } + public NodeControllerService(NCConfig config) throws Exception { this(config, getApplication(config)); } diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/task/ShutdownTask.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/task/ShutdownTask.java index c3aa5f4..e9cf3cb 100644 --- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/task/ShutdownTask.java +++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/task/ShutdownTask.java @@ -24,6 +24,7 @@ import org.apache.hyracks.control.common.base.IClusterController; import org.apache.hyracks.control.nc.NodeControllerService; +import org.apache.hyracks.util.ExitUtil; public class ShutdownTask implements Runnable { private static final Logger LOGGER = Logger.getLogger(ShutdownTask.class.getName()); @@ -46,15 +47,7 @@ // proceed with shutdown } - //run the shutdown in a new thread, so we don't block this last work task - Thread t = new Thread("NC " + ncs.getId() + " Shutdown") { - @Override - public void run() { - LOGGER.info("JVM Exiting.. Bye!"); - Runtime.getRuntime().exit(terminateNCService ? 99 : 0); - } - }; - t.start(); + ExitUtil.exit(terminateNCService ? 99 : 0); } } diff --git a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/ExitUtil.java b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/ExitUtil.java new file mode 100644 index 0000000..b039227 --- /dev/null +++ b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/ExitUtil.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.hyracks.util; + +import java.util.logging.Logger; + +@SuppressWarnings("squid:S1147") +public class ExitUtil { + + private static final Logger LOGGER = Logger.getLogger(ExitUtil.class.getName()); + + private static final ExitThread exitThread = new ExitThread(); + + private ExitUtil() { + } + + public static void init() { + // no-op, the clinit does the work + } + + public static void exit(int status) { + exitThread.setStatus(status); + exitThread.start(); + } + + private static class ExitThread extends Thread { + private int status; + + ExitThread() { + super("JVM exit thread"); + setDaemon(true); + } + + @Override + public void run() { + try { + LOGGER.info("JVM exiting with status " + status + "; bye!"); + } finally { + Runtime.getRuntime().exit(status); + } + } + + public void setStatus(int status) { + this.status = status; + } + } +} -- To view, visit https://asterix-gerrit.ics.uci.edu/1973 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-MessageType: merged Gerrit-Change-Id: I21b2090ea3ef85e95ae90de04b08b4a6d22ebe42 Gerrit-PatchSet: 2 Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Owner: Michael Blow <[email protected]> Gerrit-Reviewer: Anon. E. Moose #1000171 Gerrit-Reviewer: Jenkins <[email protected]> Gerrit-Reviewer: Michael Blow <[email protected]> Gerrit-Reviewer: Murtadha Hubail <[email protected]> Gerrit-Reviewer: Till Westmann <[email protected]> Gerrit-Reviewer: abdullah alamoudi <[email protected]>
