This is an automated email from the ASF dual-hosted git repository.
xyao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 1474e46 HDDS-5291. Handle SIGTERM to ensure clean shutdown of
OM/DN/SCM (#2301)
1474e46 is described below
commit 1474e4651f7caa8fe488982841be712b91c5f6da
Author: Bharat Viswanadham <[email protected]>
AuthorDate: Fri Jul 16 09:29:13 2021 +0530
HDDS-5291. Handle SIGTERM to ensure clean shutdown of OM/DN/SCM (#2301)
---
.../java/org/apache/hadoop/hdds/HddsUtils.java | 9 +
.../hadoop/ozone/conf/OzoneServiceConfig.java | 76 ++++
.../org/apache/hadoop/ozone/conf/package-info.java | 22 ++
.../hadoop/ozone/util/ShutdownHookManager.java | 391 +++++++++++++++++++++
.../org/apache/hadoop/ozone/util/package-info.java | 22 ++
.../apache/hadoop/ozone/HddsDatanodeService.java | 13 +-
.../transport/server/ratis/XceiverServerRatis.java | 2 +-
.../container/common/volume/MutableVolumeSet.java | 15 +-
.../hdds/scm/server/StorageContainerManager.java | 2 +
.../scm/server/StorageContainerManagerStarter.java | 12 +-
.../dist/src/shell/ozone/ozone-functions.sh | 2 +-
.../apache/hadoop/ozone/insight/LogSubcommand.java | 9 +-
.../org/apache/hadoop/ozone/om/OzoneManager.java | 8 +-
.../hadoop/ozone/om/OzoneManagerStarter.java | 12 +-
.../org/apache/hadoop/ozone/recon/ReconServer.java | 6 +-
.../java/org/apache/hadoop/ozone/s3/Gateway.java | 11 +
.../hadoop/ozone/freon/BaseFreonGenerator.java | 7 +-
.../hadoop/ozone/freon/RandomKeyGenerator.java | 16 +-
18 files changed, 604 insertions(+), 31 deletions(-)
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java
index c38ff3d..a62fd6d 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java
@@ -65,6 +65,7 @@ import static
org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_DATANODE_PORT_D
import static
org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_DATANODE_PORT_KEY;
import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_NAMES;
+import org.apache.hadoop.ozone.conf.OzoneServiceConfig;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -603,4 +604,12 @@ public final class HddsUtils {
}
return sb.toString();
}
+
+ /**
+ * Return Ozone service shutdown time out.
+ * @param conf
+ */
+ public static long getShutDownTimeOut(ConfigurationSource conf) {
+ return
conf.getObject(OzoneServiceConfig.class).getServiceShutdownTimeout();
+ }
}
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/conf/OzoneServiceConfig.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/conf/OzoneServiceConfig.java
new file mode 100644
index 0000000..58e88d3
--- /dev/null
+++
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/conf/OzoneServiceConfig.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.conf;
+
+import org.apache.hadoop.hdds.conf.Config;
+import org.apache.hadoop.hdds.conf.ConfigGroup;
+import org.apache.hadoop.hdds.conf.ConfigType;
+
+import java.util.concurrent.TimeUnit;
+
+import static org.apache.hadoop.hdds.conf.ConfigTag.DATANODE;
+import static org.apache.hadoop.hdds.conf.ConfigTag.OM;
+import static org.apache.hadoop.hdds.conf.ConfigTag.OZONE;
+import static org.apache.hadoop.hdds.conf.ConfigTag.RECON;
+import static org.apache.hadoop.hdds.conf.ConfigTag.S3GATEWAY;
+import static org.apache.hadoop.hdds.conf.ConfigTag.SCM;
+
+/**
+ * This class is used to define Ozone service level configs which are needed
+ * for all the ozone services.
+ */
+@ConfigGroup(prefix = "ozone.service")
+public class OzoneServiceConfig {
+
+ /** Minimum shutdown timeout: {@value} second(s). */
+ public static final long OZONE_SHUTDOWN_TIMEOUT_MINIMUM = 1;
+
+ /** The default time unit used: seconds. */
+ public static final TimeUnit OZONE_SHUTDOWN_TIME_UNIT_DEFAULT =
+ TimeUnit.SECONDS;
+
+ public static final int DEFAULT_SHUTDOWN_HOOK_PRIORITY = 10;
+
+ public static final String SERVICE_SHUTDOWN_TIMEOUT =
+ "shutdown.timeout";
+ /** Default shutdown hook timeout: {@value} seconds. */
+ public static final String SERVICE_SHUTDOWN_TIMEOUT_DEFAULT = "60s";
+
+ @Config(key = SERVICE_SHUTDOWN_TIMEOUT,
+ defaultValue = SERVICE_SHUTDOWN_TIMEOUT_DEFAULT,
+ type = ConfigType.TIME,
+ tags = {OZONE, OM, SCM, DATANODE, RECON, S3GATEWAY},
+ timeUnit = TimeUnit.SECONDS,
+ description = "Timeout to wait for each shutdown operation to complete" +
+ "If a hook takes longer than this time to complete, it will " +
+ "be interrupted, so the service will shutdown. This allows the " +
+ "service shutdown to recover from a blocked operation. " +
+ "The minimum duration of the timeout is 1 second, if " +
+ "hook has been configured with a timeout less than 1 second."
+ )
+ private long serviceShutdownTimeout = 60;
+
+ public long getServiceShutdownTimeout() {
+ return serviceShutdownTimeout;
+ }
+
+ public void setServiceShutdownTimeout(long serviceShutdownTimeout) {
+ this.serviceShutdownTimeout = serviceShutdownTimeout;
+ }
+}
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/conf/package-info.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/conf/package-info.java
new file mode 100644
index 0000000..cb6ec51
--- /dev/null
+++
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/conf/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Configuration classes for Ozone.
+ */
+package org.apache.hadoop.ozone.conf;
\ No newline at end of file
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/util/ShutdownHookManager.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/util/ShutdownHookManager.java
new file mode 100644
index 0000000..05dc688
--- /dev/null
+++
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/util/ShutdownHookManager.java
@@ -0,0 +1,391 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.util;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import org.apache.hadoop.hdds.HddsUtils;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.hdds.annotation.InterfaceAudience;
+import org.apache.hadoop.hdds.annotation.InterfaceStability;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeoutException;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import static
org.apache.hadoop.ozone.conf.OzoneServiceConfig.OZONE_SHUTDOWN_TIMEOUT_MINIMUM;
+import static
org.apache.hadoop.ozone.conf.OzoneServiceConfig.OZONE_SHUTDOWN_TIME_UNIT_DEFAULT;
+
+/**
+ * The <code>ShutdownHookManager</code> enables running shutdownHook
+ * in a deterministic order, higher priority first.
+ * <p/>
+ * The JVM runs ShutdownHooks in a non-deterministic order or in parallel.
+ * This class registers a single JVM shutdownHook and run all the
+ * shutdownHooks registered to it (to this class) in order based on their
+ * priority.
+ *
+ * Unless a hook was registered with a shutdown explicitly set through
+ * {@link #addShutdownHook(Runnable, int, long, TimeUnit)},
+ * the shutdown time allocated to it is set by the configuration option
+ * {@link org.apache.hadoop.ozone.conf.OzoneServiceConfig
+ * #SERVICE_SHUTDOWN_TIMEOUT}
+ * {@code ozone-site.xml}, with a default value of
+ * {@link org.apache.hadoop.ozone.conf.OzoneServiceConfig
+ * #SERVICE_SHUTDOWN_TIMEOUT_DEFAULT}
+ * seconds.
+ *
+ * This code is taken from hadoop project.
+ * 1. This is to avoid dependency on hadoop.
+ * 2. To use a ozone specific config and defaults.
+ * 3. Now any fix happened to this class in hadoop for this class, we can
+ * backport this to this with out waiting for a new hadoop release.
+ *
+ */
[email protected]
[email protected]
+public final class ShutdownHookManager {
+
+ private static final ShutdownHookManager MGR = new ShutdownHookManager();
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(ShutdownHookManager.class);
+
+
+
+ private static final ExecutorService EXECUTOR =
+ Executors.newSingleThreadExecutor(new ThreadFactoryBuilder()
+ .setDaemon(true)
+ .setNameFormat("shutdown-hook-%01d")
+ .build());
+
+ static {
+ try {
+ Runtime.getRuntime().addShutdownHook(
+ new Thread() {
+ @Override
+ public void run() {
+ if (MGR.shutdownInProgress.getAndSet(true)) {
+ LOG.info("Shutdown process invoked a second time: ignoring");
+ return;
+ }
+ long started = System.currentTimeMillis();
+ int timeoutCount = MGR.executeShutdown();
+ long ended = System.currentTimeMillis();
+ LOG.debug(String.format(
+ "Completed shutdown in %.3f seconds; Timeouts: %d",
+ (ended-started)/1000.0, timeoutCount));
+ // each of the hooks have executed; now shut down the
+ // executor itself.
+ shutdownExecutor(new OzoneConfiguration());
+ }
+ }
+ );
+ } catch (IllegalStateException ex) {
+ // JVM is being shut down. Ignore
+ LOG.warn("Failed to add the ShutdownHook", ex);
+ }
+ }
+
+ /**
+ * Execute the shutdown.
+ * This is exposed purely for testing: do not invoke it.
+ * @return the number of shutdown hooks which timed out.
+ */
+ @InterfaceAudience.Private
+ @VisibleForTesting
+ int executeShutdown() {
+ int timeouts = 0;
+ for (HookEntry entry: getShutdownHooksInOrder()) {
+ Future<?> future = EXECUTOR.submit(entry.getHook());
+ try {
+ future.get(entry.getTimeout(), entry.getTimeUnit());
+ } catch (TimeoutException ex) {
+ timeouts++;
+ future.cancel(true);
+ LOG.warn("ShutdownHook '" + entry.getHook().getClass().
+ getSimpleName() + "' timeout, " + ex.toString(), ex);
+ } catch (Throwable ex) {
+ LOG.warn("ShutdownHook '" + entry.getHook().getClass().
+ getSimpleName() + "' failed, " + ex.toString(), ex);
+ }
+ }
+ return timeouts;
+ }
+
+ /**
+ * Shutdown the executor thread itself.
+ * @param conf the configuration containing the shutdown timeout setting.
+ */
+ private static void shutdownExecutor(final ConfigurationSource conf) {
+ try {
+ EXECUTOR.shutdown();
+ long shutdownTimeout = getShutdownTimeout(conf);
+ if (!EXECUTOR.awaitTermination(
+ shutdownTimeout, OZONE_SHUTDOWN_TIME_UNIT_DEFAULT)) {
+ // timeout waiting for the
+ LOG.error("ShutdownHookManger shutdown forcefully after"
+ + " {} seconds.", shutdownTimeout);
+ EXECUTOR.shutdownNow();
+ }
+ LOG.debug("ShutdownHookManger completed shutdown.");
+ } catch (InterruptedException ex) {
+ // interrupted.
+ LOG.error("ShutdownHookManger interrupted while waiting for " +
+ "termination.", ex);
+ EXECUTOR.shutdownNow();
+ Thread.currentThread().interrupt();
+ }
+ }
+
+ /**
+ * Return <code>ShutdownHookManager</code> singleton.
+ *
+ * @return <code>ShutdownHookManager</code> singleton.
+ */
+ @InterfaceAudience.Public
+ public static ShutdownHookManager get() {
+ return MGR;
+ }
+
+ /**
+ * Get the shutdown timeout in seconds, from the supplied
+ * configuration.
+ * @param conf configuration to use.
+ * @return a timeout, always greater than or equal to
+ * {@link org.apache.hadoop.ozone.conf.OzoneServiceConfig
+ * #OZONE_SHUTDOWN_TIMEOUT_MINIMUM}
+ */
+ @InterfaceAudience.Private
+ @VisibleForTesting
+ static long getShutdownTimeout(ConfigurationSource conf) {
+ long duration = HddsUtils.getShutDownTimeOut(conf);
+ if (duration < OZONE_SHUTDOWN_TIMEOUT_MINIMUM) {
+ duration = OZONE_SHUTDOWN_TIMEOUT_MINIMUM;
+ }
+ return duration;
+ }
+
+ /**
+ * Private structure to store ShutdownHook, its priority and timeout
+ * settings.
+ */
+ @InterfaceAudience.Private
+ @VisibleForTesting
+ static class HookEntry {
+ private final Runnable hook;
+ private final int priority;
+ private final long timeout;
+ private final TimeUnit unit;
+
+ HookEntry(Runnable hook, int priority) {
+ this(hook, priority, getShutdownTimeout(new OzoneConfiguration()),
+ OZONE_SHUTDOWN_TIME_UNIT_DEFAULT);
+ }
+
+ HookEntry(Runnable hook, int priority, long timeout, TimeUnit unit) {
+ this.hook = hook;
+ this.priority = priority;
+ this.timeout = timeout;
+ this.unit = unit;
+ }
+
+ @Override
+ public int hashCode() {
+ return hook.hashCode();
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ boolean eq = false;
+ if (obj != null) {
+ if (obj instanceof HookEntry) {
+ eq = (hook == ((HookEntry)obj).hook);
+ }
+ }
+ return eq;
+ }
+
+ Runnable getHook() {
+ return hook;
+ }
+
+ int getPriority() {
+ return priority;
+ }
+
+ long getTimeout() {
+ return timeout;
+ }
+
+ TimeUnit getTimeUnit() {
+ return unit;
+ }
+ }
+
+ private final Set<HookEntry> hooks =
+ Collections.synchronizedSet(new HashSet<>());
+
+ private AtomicBoolean shutdownInProgress = new AtomicBoolean(false);
+
+ //private to constructor to ensure singularity
+ @VisibleForTesting
+ @InterfaceAudience.Private
+ ShutdownHookManager() {
+ }
+
+ /**
+ * Returns the list of shutdownHooks in order of execution,
+ * Highest priority first.
+ *
+ * @return the list of shutdownHooks in order of execution.
+ */
+ @InterfaceAudience.Private
+ @VisibleForTesting
+ List<HookEntry > getShutdownHooksInOrder() {
+ List<HookEntry > list;
+ synchronized (hooks) {
+ list = new ArrayList<HookEntry>(hooks);
+ }
+ Collections.sort(list, new Comparator< HookEntry >() {
+
+ //reversing comparison so highest priority hooks are first
+ @Override
+ public int compare(HookEntry o1, HookEntry o2) {
+ return o2.priority - o1.priority;
+ }
+ });
+ return list;
+ }
+
+ /**
+ * Adds a shutdownHook with a priority, the higher the priority
+ * the earlier will run. ShutdownHooks with same priority run
+ * in a non-deterministic order.
+ *
+ * @param shutdownHook shutdownHook <code>Runnable</code>
+ * @param priority priority of the shutdownHook.
+ */
+ @InterfaceAudience.Public
+ @InterfaceStability.Stable
+ public void addShutdownHook(Runnable shutdownHook, int priority) {
+ if (shutdownHook == null) {
+ throw new IllegalArgumentException("shutdownHook cannot be NULL");
+ }
+ if (shutdownInProgress.get()) {
+ throw new IllegalStateException("Shutdown in progress, cannot add a " +
+ "shutdownHook");
+ }
+ hooks.add(new HookEntry(shutdownHook, priority));
+ }
+
+ /**
+ *
+ * Adds a shutdownHook with a priority and timeout the higher the priority
+ * the earlier will run. ShutdownHooks with same priority run
+ * in a non-deterministic order. The shutdown hook will be terminated if it
+ * has not been finished in the specified period of time.
+ *
+ * @param shutdownHook shutdownHook <code>Runnable</code>
+ * @param priority priority of the shutdownHook
+ * @param timeout timeout of the shutdownHook
+ * @param unit unit of the timeout <code>TimeUnit</code>
+ */
+ @InterfaceAudience.Public
+ @InterfaceStability.Stable
+ public void addShutdownHook(Runnable shutdownHook, int priority, long
timeout,
+ TimeUnit unit) {
+ if (shutdownHook == null) {
+ throw new IllegalArgumentException("shutdownHook cannot be NULL");
+ }
+ if (shutdownInProgress.get()) {
+ throw new IllegalStateException("Shutdown in progress, cannot add a " +
+ "shutdownHook");
+ }
+ hooks.add(new HookEntry(shutdownHook, priority, timeout, unit));
+ }
+
+ /**
+ * Removes a shutdownHook.
+ *
+ * @param shutdownHook shutdownHook to remove.
+ * @return TRUE if the shutdownHook was registered and removed,
+ * FALSE otherwise.
+ */
+ @InterfaceAudience.Public
+ @InterfaceStability.Stable
+ public boolean removeShutdownHook(Runnable shutdownHook) {
+ if (shutdownInProgress.get()) {
+ throw new IllegalStateException("Shutdown in progress, cannot remove a "
+
+ "shutdownHook");
+ }
+ // hooks are only == by runnable
+ return hooks.remove(new HookEntry(shutdownHook, 0,
+ OZONE_SHUTDOWN_TIMEOUT_MINIMUM,
+ OZONE_SHUTDOWN_TIME_UNIT_DEFAULT));
+ }
+
+ /**
+ * Indicates if a shutdownHook is registered or not.
+ *
+ * @param shutdownHook shutdownHook to check if registered.
+ * @return TRUE/FALSE depending if the shutdownHook is is registered.
+ */
+ @InterfaceAudience.Public
+ @InterfaceStability.Stable
+ public boolean hasShutdownHook(Runnable shutdownHook) {
+ return hooks.contains(new HookEntry(shutdownHook, 0,
+ OZONE_SHUTDOWN_TIMEOUT_MINIMUM,
+ OZONE_SHUTDOWN_TIME_UNIT_DEFAULT));
+ }
+
+ /**
+ * Indicates if shutdown is in progress or not.
+ *
+ * @return TRUE if the shutdown is in progress, otherwise FALSE.
+ */
+ @InterfaceAudience.Public
+ @InterfaceStability.Stable
+ public boolean isShutdownInProgress() {
+ return shutdownInProgress.get();
+ }
+
+ /**
+ * clear all registered shutdownHooks.
+ */
+ @InterfaceAudience.Public
+ @InterfaceStability.Stable
+ public void clearShutdownHooks() {
+ hooks.clear();
+ }
+}
+
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/util/package-info.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/util/package-info.java
new file mode 100644
index 0000000..92de1ec
--- /dev/null
+++
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/util/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Utility classes required for ozone.
+ */
+package org.apache.hadoop.ozone.util;
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java
index 27a0dc8..56003e1 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java
@@ -62,6 +62,7 @@ import
org.apache.hadoop.ozone.container.common.utils.HddsVolumeUtil;
import org.apache.hadoop.ozone.container.common.volume.HddsVolume;
import org.apache.hadoop.ozone.container.common.volume.MutableVolumeSet;
import org.apache.hadoop.ozone.container.common.volume.StorageVolume;
+import org.apache.hadoop.ozone.util.ShutdownHookManager;
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
import
org.apache.hadoop.security.authentication.client.AuthenticationException;
@@ -71,11 +72,14 @@ import org.apache.hadoop.util.Time;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import com.sun.jmx.mbeanserver.Introspector;
+
import static
org.apache.hadoop.hdds.security.x509.certificate.utils.CertificateCodec.getX509Certificate;
import static
org.apache.hadoop.hdds.security.x509.certificates.utils.CertificateSignRequest.getEncodedString;
import static
org.apache.hadoop.ozone.OzoneConfigKeys.HDDS_DATANODE_PLUGINS_KEY;
+import static
org.apache.hadoop.ozone.conf.OzoneServiceConfig.DEFAULT_SHUTDOWN_HOOK_PRIORITY;
import static org.apache.hadoop.ozone.common.Storage.StorageState.INITIALIZED;
import static org.apache.hadoop.util.ExitUtil.terminate;
+
import org.bouncycastle.pkcs.PKCS10CertificationRequest;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -169,7 +173,14 @@ public class HddsDatanodeService extends GenericCli
implements ServicePlugin {
HddsDatanodeService.class, args, LOG);
}
start(createOzoneConfiguration());
- join();
+ ShutdownHookManager.get().addShutdownHook(() -> {
+ try {
+ stop();
+ join();
+ } catch (Exception e) {
+ LOG.error("Error during stop Ozone Datanode.", e);
+ }
+ }, DEFAULT_SHUTDOWN_HOOK_PRIORITY);
return null;
}
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java
index 867127e..cca1c08 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java
@@ -513,7 +513,7 @@ public final class XceiverServerRatis implements
XceiverServerSpi {
}
isStarted = false;
} catch (IOException e) {
- throw new RuntimeException(e);
+ LOG.error("XceiverServerRatis Could not be stopped gracefully.", e);
}
}
}
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/MutableVolumeSet.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/MutableVolumeSet.java
index 76a576b..47fe4e3 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/MutableVolumeSet.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/MutableVolumeSet.java
@@ -39,13 +39,13 @@ import
org.apache.hadoop.ozone.container.common.statemachine.DatanodeConfigurati
import org.apache.hadoop.ozone.container.common.statemachine.StateContext;
import org.apache.hadoop.util.DiskChecker.DiskOutOfSpaceException;
import org.apache.hadoop.util.ShutdownHookManager;
+import org.apache.hadoop.util.Timer;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import static org.apache.hadoop.util.RunJar.SHUTDOWN_HOOK_PRIORITY;
-import org.apache.hadoop.util.Timer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -415,15 +415,17 @@ public class MutableVolumeSet implements VolumeSet {
* This method, call shutdown on each volume to shutdown volume usage
* thread and write scmUsed on each volume.
*/
- private void saveVolumeSetUsed() {
- for (StorageVolume volume : volumeMap.values()) {
+
+ private synchronized void saveVolumeSetUsed() {
+ for (StorageVolume hddsVolume : volumeMap.values()) {
try {
- volume.shutdown();
+ hddsVolume.shutdown();
} catch (Exception ex) {
- LOG.error("Failed to shutdown volume : " + volume.getStorageDir(),
+ LOG.error("Failed to shutdown volume : " + hddsVolume.getStorageDir(),
ex);
}
}
+ volumeMap.clear();
}
/**
@@ -431,9 +433,6 @@ public class MutableVolumeSet implements VolumeSet {
*/
public void shutdown() {
saveVolumeSetUsed();
- if (shutdownHook != null) {
- ShutdownHookManager.get().removeShutdownHook(shutdownHook);
- }
}
@Override
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
index ad011e7..c739757 100644
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
@@ -1423,6 +1423,7 @@ public final class StorageContainerManager extends
ServiceRuntimeInfoImpl
}
try {
+ LOG.info("Stopping SCM HA services.");
scmHAManager.shutdown();
} catch (Exception ex) {
LOG.error("SCM HA Manager stop failed", ex);
@@ -1432,6 +1433,7 @@ public final class StorageContainerManager extends
ServiceRuntimeInfoImpl
IOUtils.cleanupWithLogger(LOG, pipelineManager);
try {
+ LOG.info("Stopping SCM MetadataStore.");
scmMetadataStore.stop();
} catch (Exception ex) {
LOG.error("SCM Metadata store stop failed", ex);
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManagerStarter.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManagerStarter.java
index 8960269..1d8859f 100644
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManagerStarter.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManagerStarter.java
@@ -28,6 +28,7 @@ import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.tracing.TracingUtil;
import org.apache.hadoop.hdds.utils.HddsVersionInfo;
import org.apache.hadoop.ozone.common.StorageInfo;
+import org.apache.hadoop.ozone.util.ShutdownHookManager;
import
org.apache.hadoop.security.authentication.client.AuthenticationException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -36,6 +37,8 @@ import picocli.CommandLine.Command;
import java.io.IOException;
+import static
org.apache.hadoop.ozone.conf.OzoneServiceConfig.DEFAULT_SHUTDOWN_HOOK_PRIORITY;
+
/**
* This class provides a command line interface to start the SCM
* using Picocli.
@@ -164,7 +167,14 @@ public class StorageContainerManagerStarter extends
GenericCli {
public void start(OzoneConfiguration conf) throws Exception {
StorageContainerManager stm = StorageContainerManager.createSCM(conf);
stm.start();
- stm.join();
+ ShutdownHookManager.get().addShutdownHook(() -> {
+ try {
+ stm.stop();
+ stm.join();
+ } catch (Exception e) {
+ LOG.error("Error during stop StorageContainerManager", e);
+ }
+ }, DEFAULT_SHUTDOWN_HOOK_PRIORITY);
}
@Override
diff --git a/hadoop-ozone/dist/src/shell/ozone/ozone-functions.sh
b/hadoop-ozone/dist/src/shell/ozone/ozone-functions.sh
index 886b084..91ede8d 100755
--- a/hadoop-ozone/dist/src/shell/ozone/ozone-functions.sh
+++ b/hadoop-ozone/dist/src/shell/ozone/ozone-functions.sh
@@ -848,7 +848,7 @@ function ozone_basic_init
OZONE_LOGFILE=${OZONE_LOGFILE:-ozone.log}
OZONE_LOGLEVEL=${OZONE_LOGLEVEL:-INFO}
OZONE_NICENESS=${OZONE_NICENESS:-0}
- OZONE_STOP_TIMEOUT=${OZONE_STOP_TIMEOUT:-5}
+ OZONE_STOP_TIMEOUT=${OZONE_STOP_TIMEOUT:-60}
OZONE_PID_DIR=${OZONE_PID_DIR:-/tmp}
OZONE_ROOT_LOGGER=${OZONE_ROOT_LOGGER:-${OZONE_LOGLEVEL},console}
OZONE_DAEMON_ROOT_LOGGER=${OZONE_DAEMON_ROOT_LOGGER:-${OZONE_LOGLEVEL},RFA}
diff --git
a/hadoop-ozone/insight/src/main/java/org/apache/hadoop/ozone/insight/LogSubcommand.java
b/hadoop-ozone/insight/src/main/java/org/apache/hadoop/ozone/insight/LogSubcommand.java
index 5e487a5..1bae30b 100644
---
a/hadoop-ozone/insight/src/main/java/org/apache/hadoop/ozone/insight/LogSubcommand.java
+++
b/hadoop-ozone/insight/src/main/java/org/apache/hadoop/ozone/insight/LogSubcommand.java
@@ -36,12 +36,15 @@ import org.apache.hadoop.hdds.cli.HddsVersionProvider;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.ozone.insight.LoggerSource.Level;
+import org.apache.hadoop.ozone.util.ShutdownHookManager;
import org.apache.http.HttpResponse;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.HttpClientBuilder;
import picocli.CommandLine;
+import static
org.apache.hadoop.ozone.conf.OzoneServiceConfig.DEFAULT_SHUTDOWN_HOOK_PRIORITY;
+
/**
* Subcommand to display log.
*/
@@ -76,9 +79,9 @@ public class LogSubcommand extends BaseInsightSubCommand
List<LoggerSource> loggers = insight.getRelatedLoggers(verbose, filters);
setLogLevels(conf, loggers, LoggerSource::getLevel);
- Runtime.getRuntime().addShutdownHook(new Thread(() ->
- setLogLevels(conf, loggers, any -> Level.INFO)
- ));
+ ShutdownHookManager.get().addShutdownHook(() ->
+ setLogLevels(conf, loggers, any -> Level.INFO),
+ DEFAULT_SHUTDOWN_HOOK_PRIORITY);
Set<Component> sources = loggers.stream().map(LoggerSource::getComponent)
.collect(Collectors.toSet());
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
index 19aee4c..82eb2f8 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
@@ -169,6 +169,7 @@ import org.apache.hadoop.ozone.upgrade.UpgradeFinalizer;
import org.apache.hadoop.ozone.upgrade.UpgradeFinalizer.StatusAndMessages;
import org.apache.hadoop.hdds.ExitManager;
import org.apache.hadoop.ozone.util.OzoneVersionInfo;
+import org.apache.hadoop.ozone.util.ShutdownHookManager;
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod;
@@ -178,7 +179,6 @@ import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.util.JvmPauseMonitor;
import org.apache.hadoop.util.KMSUtil;
import org.apache.hadoop.util.ReflectionUtils;
-import org.apache.hadoop.util.ShutdownHookManager;
import org.apache.hadoop.util.Time;
import com.fasterxml.jackson.databind.ObjectMapper;
@@ -705,8 +705,10 @@ public final class OzoneManager extends
ServiceRuntimeInfoImpl
private void saveOmMetrics() {
try {
boolean success;
- Files.createDirectories(
- getTempMetricsStorageFile().getParentFile().toPath());
+ File parent = getTempMetricsStorageFile().getParentFile();
+ if (!parent.exists()) {
+ Files.createDirectories(parent.toPath());
+ }
try (BufferedWriter writer = new BufferedWriter(
new OutputStreamWriter(new FileOutputStream(
getTempMetricsStorageFile()), StandardCharsets.UTF_8))) {
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManagerStarter.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManagerStarter.java
index d6a9550..39069a8 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManagerStarter.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManagerStarter.java
@@ -23,6 +23,7 @@ import org.apache.hadoop.hdds.cli.HddsVersionProvider;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.tracing.TracingUtil;
import org.apache.hadoop.ozone.util.OzoneVersionInfo;
+import org.apache.hadoop.ozone.util.ShutdownHookManager;
import
org.apache.hadoop.security.authentication.client.AuthenticationException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -32,6 +33,8 @@ import picocli.CommandLine.Command;
import java.io.IOException;
+import static
org.apache.hadoop.ozone.conf.OzoneServiceConfig.DEFAULT_SHUTDOWN_HOOK_PRIORITY;
+
/**
* This class provides a command line interface to start the OM
* using Picocli.
@@ -146,7 +149,14 @@ public class OzoneManagerStarter extends GenericCli {
AuthenticationException {
OzoneManager om = OzoneManager.createOm(conf);
om.start();
- om.join();
+ ShutdownHookManager.get().addShutdownHook(() -> {
+ try {
+ om.stop();
+ om.join();
+ } catch (Exception e) {
+ LOG.error("Error during stop OzoneManager.", e);
+ }
+ }, DEFAULT_SHUTDOWN_HOOK_PRIORITY);
}
@Override
diff --git
a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconServer.java
b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconServer.java
index c9030c0..d670a6d 100644
---
a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconServer.java
+++
b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconServer.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.ozone.recon;
import static
org.apache.hadoop.hdds.recon.ReconConfig.ConfigStrings.OZONE_RECON_KERBEROS_KEYTAB_FILE_KEY;
import static
org.apache.hadoop.hdds.recon.ReconConfig.ConfigStrings.OZONE_RECON_KERBEROS_PRINCIPAL_KEY;
+import static
org.apache.hadoop.ozone.conf.OzoneServiceConfig.DEFAULT_SHUTDOWN_HOOK_PRIORITY;
import org.apache.hadoop.hdds.HddsUtils;
import org.apache.hadoop.hdds.StringUtils;
@@ -35,6 +36,7 @@ import
org.apache.hadoop.ozone.recon.spi.ReconNamespaceSummaryManager;
import org.apache.hadoop.ozone.recon.spi.StorageContainerServiceProvider;
import org.apache.hadoop.ozone.recon.spi.impl.ReconDBProvider;
import org.apache.hadoop.ozone.util.OzoneVersionInfo;
+import org.apache.hadoop.ozone.util.ShutdownHookManager;
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
import
org.apache.hadoop.security.authentication.client.AuthenticationException;
@@ -122,14 +124,14 @@ public class ReconServer extends GenericCli {
start();
isStarted = true;
- Runtime.getRuntime().addShutdownHook(new Thread(() -> {
+ ShutdownHookManager.get().addShutdownHook(() -> {
try {
stop();
join();
} catch (Exception e) {
LOG.error("Error during stop Recon server", e);
}
- }));
+ }, DEFAULT_SHUTDOWN_HOOK_PRIORITY);
return null;
}
diff --git
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/Gateway.java
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/Gateway.java
index 459a7a4..8b5eddb 100644
---
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/Gateway.java
+++
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/Gateway.java
@@ -26,11 +26,14 @@ import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.tracing.TracingUtil;
import org.apache.hadoop.ozone.util.OzoneVersionInfo;
+import org.apache.hadoop.ozone.util.ShutdownHookManager;
import org.apache.hadoop.security.UserGroupInformation;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import picocli.CommandLine.Command;
+import static
org.apache.hadoop.ozone.conf.OzoneServiceConfig.DEFAULT_SHUTDOWN_HOOK_PRIORITY;
+
/**
* This class is used to start/stop S3 compatible rest server.
*/
@@ -56,6 +59,14 @@ public class Gateway extends GenericCli {
UserGroupInformation.setConfiguration(ozoneConfiguration);
httpServer = new S3GatewayHttpServer(ozoneConfiguration, "s3gateway");
start();
+
+ ShutdownHookManager.get().addShutdownHook(() -> {
+ try {
+ stop();
+ } catch (Exception e) {
+ LOG.error("Error during stop S3Gateway", e);
+ }
+ }, DEFAULT_SHUTDOWN_HOOK_PRIORITY);
return null;
}
diff --git
a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/BaseFreonGenerator.java
b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/BaseFreonGenerator.java
index 0649a0c..0a639ec 100644
---
a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/BaseFreonGenerator.java
+++
b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/BaseFreonGenerator.java
@@ -43,6 +43,7 @@ import org.apache.hadoop.ozone.om.protocolPB.OmTransport;
import org.apache.hadoop.ozone.om.protocolPB.OmTransportFactory;
import
org.apache.hadoop.ozone.om.protocolPB.OzoneManagerProtocolClientSideTranslatorPB;
import org.apache.hadoop.ozone.om.protocolPB.OzoneManagerProtocolPB;
+import org.apache.hadoop.ozone.util.ShutdownHookManager;
import org.apache.hadoop.security.UserGroupInformation;
import com.codahale.metrics.ConsoleReporter;
@@ -251,15 +252,15 @@ public class BaseFreonGenerator {
pathSchema = new PathSchema(prefix);
- Runtime.getRuntime().addShutdownHook(
- new Thread(() -> {
+ ShutdownHookManager.get().addShutdownHook(
+ () -> {
try {
freonCommand.stopHttpServer();
} catch (Exception ex) {
LOG.error("HTTP server can't be stopped.", ex);
}
printReport();
- }));
+ }, 10);
executor = Executors.newFixedThreadPool(threadNo);
diff --git
a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/RandomKeyGenerator.java
b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/RandomKeyGenerator.java
index a9f0f51..4b2e40d 100644
---
a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/RandomKeyGenerator.java
+++
b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/RandomKeyGenerator.java
@@ -53,6 +53,7 @@ import org.apache.hadoop.ozone.client.OzoneClientFactory;
import org.apache.hadoop.ozone.client.OzoneVolume;
import org.apache.hadoop.ozone.client.io.OzoneInputStream;
import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
+import org.apache.hadoop.ozone.util.ShutdownHookManager;
import org.apache.hadoop.util.Time;
import org.apache.hadoop.util.VersionInfo;
@@ -74,6 +75,8 @@ import picocli.CommandLine.Command;
import picocli.CommandLine.Option;
import picocli.CommandLine.ParentCommand;
+import static
org.apache.hadoop.ozone.conf.OzoneServiceConfig.DEFAULT_SHUTDOWN_HOOK_PRIORITY;
+
/**
* Data generator tool to generate as much keys as possible.
*/
@@ -381,13 +384,12 @@ public final class RandomKeyGenerator implements
Callable<Void> {
* Adds ShutdownHook to print statistics.
*/
private void addShutdownHook() {
- Runtime.getRuntime().addShutdownHook(
- new Thread(() -> {
- printStats(System.out);
- if (freon != null) {
- freon.stopHttpServer();
- }
- }));
+ ShutdownHookManager.get().addShutdownHook(() -> {
+ printStats(System.out);
+ if (freon != null) {
+ freon.stopHttpServer();
+ }
+ }, DEFAULT_SHUTDOWN_HOOK_PRIORITY);
}
private void doCleanObjects() throws InterruptedException {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
