This is an automated email from the ASF dual-hosted git repository.
sumitagrawal pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 153659032b HDDS-8782. Improve Volume Scanner Health checks. (#4867)
153659032b is described below
commit 153659032bcdc177f7a0e0afc6ff97c06fc2a76a
Author: Ethan Rose <[email protected]>
AuthorDate: Wed Jun 28 01:30:56 2023 -0700
HDDS-8782. Improve Volume Scanner Health checks. (#4867)
---
.../common/statemachine/DatanodeConfiguration.java | 119 +++++++++++-
.../container/common/utils/DiskCheckUtil.java | 201 +++++++++++++++++++++
.../ozone/container/common/volume/HddsVolume.java | 30 ++-
.../container/common/volume/StorageVolume.java | 151 +++++++++++++++-
.../common/volume/StorageVolumeChecker.java | 9 +-
.../container/common/utils/TestDiskCheckUtil.java | 88 +++++++++
.../container/common/volume/TestHddsVolume.java | 66 ++++++-
.../container/common/volume/TestStorageVolume.java | 185 +++++++++++++++++++
8 files changed, 826 insertions(+), 23 deletions(-)
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeConfiguration.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeConfiguration.java
index a52f941358..164af7f31c 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeConfiguration.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeConfiguration.java
@@ -42,6 +42,12 @@ public class DatanodeConfiguration {
"hdds.datanode.container.delete.threads.max";
static final String PERIODIC_DISK_CHECK_INTERVAL_MINUTES_KEY =
"hdds.datanode.periodic.disk.check.interval.minutes";
+ public static final String DISK_CHECK_FILE_SIZE_KEY =
+ "hdds.datanode.disk.check.file.size";
+ public static final String DISK_CHECK_IO_TEST_COUNT_KEY =
+ "hdds.datanode.disk.check.io.test.count";
+ public static final String DISK_CHECK_IO_FAILURES_TOLERATED_KEY =
+ "hdds.datanode.disk.check.io.failures.tolerated";
public static final String FAILED_DATA_VOLUMES_TOLERATED_KEY =
"hdds.datanode.failed.data.volumes.tolerated";
public static final String FAILED_METADATA_VOLUMES_TOLERATED_KEY =
@@ -64,10 +70,16 @@ public class DatanodeConfiguration {
static final int FAILED_VOLUMES_TOLERATED_DEFAULT = -1;
+ public static final int DISK_CHECK_IO_TEST_COUNT_DEFAULT = 3;
+
+ public static final int DISK_CHECK_IO_FAILURES_TOLERATED_DEFAULT = 1;
+
+ public static final int DISK_CHECK_FILE_SIZE_DEFAULT = 100;
+
static final boolean WAIT_ON_ALL_FOLLOWERS_DEFAULT = false;
static final long DISK_CHECK_MIN_GAP_DEFAULT =
- Duration.ofMinutes(15).toMillis();
+ Duration.ofMinutes(10).toMillis();
static final long DISK_CHECK_TIMEOUT_DEFAULT =
Duration.ofMinutes(10).toMillis();
@@ -265,8 +277,44 @@ public class DatanodeConfiguration {
)
private int failedDbVolumesTolerated = FAILED_VOLUMES_TOLERATED_DEFAULT;
+ @Config(key = "disk.check.io.test.count",
+ defaultValue = "3",
+ type = ConfigType.INT,
+ tags = { DATANODE },
+ description = "The number of IO tests required to determine if a disk " +
+ " has failed. Each disk check does one IO test. The volume will be "
+
+ "failed if more than " +
+ "hdds.datanode.disk.check.io.failures.tolerated out of the last " +
+ "hdds.datanode.disk.check.io.test.count runs failed. Set to 0 " +
+ "to disable disk IO checks."
+ )
+ private int volumeIOTestCount = DISK_CHECK_IO_TEST_COUNT_DEFAULT;
+
+ @Config(key = "disk.check.io.failures.tolerated",
+ defaultValue = "1",
+ type = ConfigType.INT,
+ tags = { DATANODE },
+ description = "The number of IO tests out of the last hdds.datanode" +
+ ".disk.check.io.test.count test run that are allowed to fail before"
+
+ " the volume is marked as failed."
+ )
+ private int volumeIOFailureTolerance =
+ DISK_CHECK_IO_FAILURES_TOLERATED_DEFAULT;
+
+ @Config(key = "disk.check.file.size",
+ defaultValue = "100B",
+ type = ConfigType.SIZE,
+ tags = { DATANODE },
+ description = "The size of the temporary file that will be synced to " +
+ "the disk and " +
+ "read back to assess its health. The contents of the " +
+ "file will be stored in memory during the duration of the check."
+ )
+ private int volumeHealthCheckFileSize =
+ DISK_CHECK_FILE_SIZE_DEFAULT;
+
@Config(key = "disk.check.min.gap",
- defaultValue = "15m",
+ defaultValue = "10m",
type = ConfigType.TIME,
tags = { DATANODE },
description = "The minimum gap between two successive checks of the same"
@@ -461,6 +509,48 @@ public class DatanodeConfiguration {
failedDbVolumesTolerated = FAILED_VOLUMES_TOLERATED_DEFAULT;
}
+ if (volumeIOTestCount == 0) {
+ LOG.info("{} set to {}. Disk IO health tests have been disabled.",
+ DISK_CHECK_IO_TEST_COUNT_KEY, volumeIOTestCount);
+ } else {
+ if (volumeIOTestCount < 0) {
+ LOG.warn("{} must be greater than 0 but was set to {}." +
+ "Defaulting to {}",
+ DISK_CHECK_IO_TEST_COUNT_KEY, volumeIOTestCount,
+ DISK_CHECK_IO_TEST_COUNT_DEFAULT);
+ volumeIOTestCount = DISK_CHECK_IO_TEST_COUNT_DEFAULT;
+ }
+
+ if (volumeIOFailureTolerance < 0) {
+ LOG.warn("{} must be greater than or equal to 0 but was set to {}. " +
+ "Defaulting to {}",
+ DISK_CHECK_IO_FAILURES_TOLERATED_KEY, volumeIOFailureTolerance,
+ DISK_CHECK_IO_FAILURES_TOLERATED_DEFAULT);
+ volumeIOFailureTolerance = DISK_CHECK_IO_FAILURES_TOLERATED_DEFAULT;
+ }
+
+ if (volumeIOFailureTolerance >= volumeIOTestCount) {
+ LOG.warn("{} was set to {} but cannot be greater or equals to {} " +
+ "set to {}. Defaulting {} to {} and {} to {}",
+ DISK_CHECK_IO_FAILURES_TOLERATED_KEY, volumeIOFailureTolerance,
+ DISK_CHECK_IO_TEST_COUNT_KEY, volumeIOTestCount,
+ DISK_CHECK_IO_FAILURES_TOLERATED_KEY,
+ DISK_CHECK_IO_FAILURES_TOLERATED_DEFAULT,
+ DISK_CHECK_IO_TEST_COUNT_KEY, DISK_CHECK_IO_TEST_COUNT_DEFAULT);
+ volumeIOTestCount = DISK_CHECK_IO_TEST_COUNT_DEFAULT;
+ volumeIOFailureTolerance = DISK_CHECK_IO_FAILURES_TOLERATED_DEFAULT;
+ }
+
+ if (volumeHealthCheckFileSize < 1) {
+ LOG.warn(DISK_CHECK_FILE_SIZE_KEY +
+ "must be at least 1 byte and was set to {}. Defaulting to {}",
+ volumeHealthCheckFileSize,
+ DISK_CHECK_FILE_SIZE_DEFAULT);
+ volumeHealthCheckFileSize =
+ DISK_CHECK_FILE_SIZE_DEFAULT;
+ }
+ }
+
if (diskCheckMinGap < 0) {
LOG.warn(DISK_CHECK_MIN_GAP_KEY +
" must be greater than zero and was set to {}. Defaulting to {}",
@@ -497,7 +587,6 @@ public class DatanodeConfiguration {
rocksdbDeleteObsoleteFilesPeriod =
ROCKSDB_DELETE_OBSOLETE_FILES_PERIOD_MICRO_SECONDS_DEFAULT;
}
-
}
public void setContainerDeleteThreads(int containerDeleteThreads) {
@@ -541,6 +630,30 @@ public class DatanodeConfiguration {
this.failedDbVolumesTolerated = failedVolumesTolerated;
}
+ public int getVolumeIOTestCount() {
+ return volumeIOTestCount;
+ }
+
+ public void setVolumeIOTestCount(int testCount) {
+ this.volumeIOTestCount = testCount;
+ }
+
+ public int getVolumeIOFailureTolerance() {
+ return volumeIOFailureTolerance;
+ }
+
+ public void setVolumeIOFailureTolerance(int failureTolerance) {
+ volumeIOFailureTolerance = failureTolerance;
+ }
+
+ public int getVolumeHealthCheckFileSize() {
+ return volumeHealthCheckFileSize;
+ }
+
+ public void getVolumeHealthCheckFileSize(int fileSizeBytes) {
+ this.volumeHealthCheckFileSize = fileSizeBytes;
+ }
+
public boolean getCheckEmptyContainerDir() {
return bCheckEmptyContainerDir;
}
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/utils/DiskCheckUtil.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/utils/DiskCheckUtil.java
new file mode 100644
index 0000000000..b267b1d479
--- /dev/null
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/utils/DiskCheckUtil.java
@@ -0,0 +1,201 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.container.common.utils;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.SyncFailedException;
+import java.util.Arrays;
+import java.util.Random;
+import java.util.UUID;
+
+/**
+ * Utility class that supports checking disk health when provided a directory
+ * where the disk is mounted.
+ */
+public final class DiskCheckUtil {
+ private DiskCheckUtil() { }
+
+ // For testing purposes, an alternate check implementation can be provided
+ // to inject failures.
+ private static DiskChecks impl = new DiskChecksImpl();
+
+ @VisibleForTesting
+ public static void setTestImpl(DiskChecks diskChecks) {
+ impl = diskChecks;
+ }
+
+ @VisibleForTesting
+ public static void clearTestImpl() {
+ impl = new DiskChecksImpl();
+ }
+
+ public static boolean checkExistence(File storageDir) {
+ return impl.checkExistence(storageDir);
+ }
+
+ public static boolean checkPermissions(File storageDir) {
+ return impl.checkPermissions(storageDir);
+ }
+
+ public static boolean checkReadWrite(File storageDir, File testFileDir,
+ int numBytesToWrite) {
+ return impl.checkReadWrite(storageDir, testFileDir, numBytesToWrite);
+ }
+
+ /**
+ * Defines operations that must be implemented by a class injecting
+ * failures into this class. Default implementations return true so that
+ * tests only need to override methods for the failures they want to test.
+ */
+ public interface DiskChecks {
+ default boolean checkExistence(File storageDir) {
+ return true;
+ }
+ default boolean checkPermissions(File storageDir) {
+ return true;
+ }
+ default boolean checkReadWrite(File storageDir, File testFileDir,
+ int numBytesToWrite) {
+ return true;
+ }
+ }
+
+ /**
+ * The default implementation of DiskCheck that production code will use
+ * for disk checking.
+ */
+ private static class DiskChecksImpl implements DiskChecks {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(DiskCheckUtil.class);
+
+ private static final Random RANDOM = new Random();
+
+ @Override
+ public boolean checkExistence(File diskDir) {
+ if (!diskDir.exists()) {
+ logError(diskDir, "Directory does not exist.");
+ return false;
+ }
+ return true;
+ }
+
+ @Override
+ public boolean checkPermissions(File storageDir) {
+ // Check all permissions on the volume. If there are multiple permission
+ // errors, count it as one failure so the admin can fix them all at once.
+ boolean permissionsCorrect = true;
+ if (!storageDir.canRead()) {
+ logError(storageDir,
+ "Datanode does not have read permission on volume.");
+ permissionsCorrect = false;
+ }
+ if (!storageDir.canWrite()) {
+ logError(storageDir,
+ "Datanode does not have write permission on volume.");
+ permissionsCorrect = false;
+ }
+ if (!storageDir.canExecute()) {
+ logError(storageDir, "Datanode does not have execute" +
+ "permission on volume.");
+ permissionsCorrect = false;
+ }
+
+ return permissionsCorrect;
+ }
+
+ @Override
+ public boolean checkReadWrite(File storageDir,
+ File testFileDir, int numBytesToWrite) {
+ File testFile = new File(testFileDir, "disk-check-" + UUID.randomUUID());
+ byte[] writtenBytes = new byte[numBytesToWrite];
+ RANDOM.nextBytes(writtenBytes);
+ try (FileOutputStream fos = new FileOutputStream(testFile)) {
+ fos.write(writtenBytes);
+ fos.getFD().sync();
+ } catch (FileNotFoundException notFoundEx) {
+ logError(storageDir, String.format("Could not find file %s for " +
+ "volume check.", testFile), notFoundEx);
+ return false;
+ } catch (SyncFailedException syncEx) {
+ logError(storageDir, String.format("Could sync file %s to disk.",
+ testFile), syncEx);
+ return false;
+ } catch (IOException ioEx) {
+ logError(storageDir, String.format("Could not write file %s " +
+ "for volume check.", testFile), ioEx);
+ return false;
+ }
+
+ // Read data back from the test file.
+ byte[] readBytes = new byte[numBytesToWrite];
+ try (FileInputStream fis = new FileInputStream(testFile)) {
+ int numBytesRead = fis.read(readBytes);
+ if (numBytesRead != numBytesToWrite) {
+ logError(storageDir, String.format("%d bytes written to file %s " +
+ "but %d bytes were read back.", numBytesToWrite, testFile,
+ numBytesRead));
+ return false;
+ }
+ } catch (FileNotFoundException notFoundEx) {
+ logError(storageDir, String.format("Could not find file %s " +
+ "for volume check.", testFile), notFoundEx);
+ return false;
+ } catch (IOException ioEx) {
+ logError(storageDir, String.format("Could not read file %s " +
+ "for volume check.", testFile), ioEx);
+ return false;
+ }
+
+ // Check that test file has the expected content.
+ if (!Arrays.equals(writtenBytes, readBytes)) {
+ logError(storageDir, String.format("%d Bytes read from file " +
+ "%s do not match the %d bytes that were written.",
+ writtenBytes.length, testFile, readBytes.length));
+ return false;
+ }
+
+ // Delete the file.
+ if (!testFile.delete()) {
+ logError(storageDir, String.format("Could not delete file %s " +
+ "for volume check.", testFile));
+ return false;
+ }
+
+ // If all checks passed, the volume is healthy.
+ return true;
+ }
+
+ private void logError(File storageDir, String message) {
+ LOG.error("Volume {} failed health check. {}", storageDir, message);
+ }
+
+ private void logError(File storageDir, String message, Exception ex) {
+ LOG.error("Volume {} failed health check. {}", storageDir, message, ex);
+ }
+ }
+}
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/HddsVolume.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/HddsVolume.java
index bcbfe37554..2a2de08cb2 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/HddsVolume.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/HddsVolume.java
@@ -310,20 +310,32 @@ public class HddsVolume extends StorageVolume {
}
@Override
- public VolumeCheckResult check(@Nullable Boolean unused) throws Exception {
+ public synchronized VolumeCheckResult check(@Nullable Boolean unused)
+ throws Exception {
VolumeCheckResult result = super.check(unused);
- if (!isDbLoaded()) {
- return result;
- }
+
DatanodeConfiguration df =
getConf().getObject(DatanodeConfiguration.class);
if (result != VolumeCheckResult.HEALTHY ||
- !df.getContainerSchemaV3Enabled() || !df.autoCompactionSmallSstFile())
{
+ !df.getContainerSchemaV3Enabled() || !isDbLoaded()) {
return result;
}
- // Calculate number of files per level and size per level
- RawDB rawDB = DatanodeStoreCache.getInstance().getDB(
- new File(dbParentDir, CONTAINER_DB_NAME).getAbsolutePath(), getConf());
- rawDB.getStore().compactionIfNeeded();
+
+ // Check that per-volume RocksDB is present.
+ File dbFile = new File(dbParentDir, CONTAINER_DB_NAME);
+ if (!dbFile.exists() || !dbFile.canRead()) {
+ LOG.warn("Volume {} failed health check. Could not access RocksDB at " +
+ "{}", getStorageDir(), dbFile);
+ return VolumeCheckResult.FAILED;
+ }
+
+ // TODO HDDS-8784 trigger compaction outside of volume check. Then the
+ // exception can be removed.
+ if (df.autoCompactionSmallSstFile()) {
+ // Calculate number of files per level and size per level
+ RawDB rawDB = DatanodeStoreCache.getInstance().getDB(
+ dbFile.getAbsolutePath(), getConf());
+ rawDB.getStore().compactionIfNeeded();
+ }
return VolumeCheckResult.HEALTHY;
}
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/StorageVolume.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/StorageVolume.java
index 8141c57055..95d1b2c2de 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/StorageVolume.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/StorageVolume.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.ozone.container.common.volume;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.hdds.conf.ConfigurationSource;
@@ -27,8 +28,9 @@ import
org.apache.hadoop.hdfs.server.datanode.checker.Checkable;
import org.apache.hadoop.hdfs.server.datanode.checker.VolumeCheckResult;
import org.apache.hadoop.ozone.common.InconsistentStorageStateException;
import org.apache.hadoop.ozone.container.common.helpers.DatanodeVersionFile;
+import
org.apache.hadoop.ozone.container.common.statemachine.DatanodeConfiguration;
+import org.apache.hadoop.ozone.container.common.utils.DiskCheckUtil;
import org.apache.hadoop.ozone.container.common.utils.StorageVolumeUtil;
-import org.apache.hadoop.util.DiskChecker;
import org.apache.hadoop.util.Time;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -37,10 +39,15 @@ import javax.annotation.Nullable;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.LinkedList;
import java.util.Objects;
import java.util.Optional;
import java.util.Properties;
+import java.util.Queue;
import java.util.UUID;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Stream;
import static
org.apache.hadoop.ozone.container.common.HDDSVolumeLayoutVersion.getLatestVersion;
@@ -66,6 +73,9 @@ public abstract class StorageVolume
// The name of the directory used for temporary files on the volume.
public static final String TMP_DIR_NAME = "tmp";
+ // The name of the directory where temporary files used to check disk
+ // health are written to. This will go inside the tmp directory.
+ public static final String TMP_DISK_CHECK_DIR_NAME = "disk-check";
/**
* Type for StorageVolume.
@@ -111,11 +121,22 @@ public abstract class StorageVolume
private final File storageDir;
private String workingDirName;
private File tmpDir;
+ private File diskCheckDir;
private final Optional<VolumeInfo> volumeInfo;
private final VolumeSet volumeSet;
+ /*
+ Fields used to implement IO based disk health checks.
+ If more than ioFailureTolerance IO checks fail out of the last ioTestCount
+ tests run, then the volume is considered failed.
+ */
+ private final int ioTestCount;
+ private final int ioFailureTolerance;
+ private AtomicInteger currentIOFailureCount;
+ private Queue<Boolean> ioTestSlidingWindow;
+ private int healthCheckFileSize;
protected StorageVolume(Builder<?> b) throws IOException {
if (!b.failedVolume) {
@@ -131,12 +152,22 @@ public abstract class StorageVolume
this.clusterID = b.clusterID;
this.datanodeUuid = b.datanodeUuid;
this.conf = b.conf;
+
+ DatanodeConfiguration dnConf =
+ conf.getObject(DatanodeConfiguration.class);
+ this.ioTestCount = dnConf.getVolumeIOTestCount();
+ this.ioFailureTolerance = dnConf.getVolumeIOFailureTolerance();
+ this.ioTestSlidingWindow = new LinkedList<>();
+ this.currentIOFailureCount = new AtomicInteger(0);
+ this.healthCheckFileSize = dnConf.getVolumeHealthCheckFileSize();
} else {
storageDir = new File(b.volumeRootStr);
this.volumeInfo = Optional.empty();
this.volumeSet = null;
this.storageID = UUID.randomUUID().toString();
this.state = VolumeState.FAILED;
+ this.ioTestCount = 0;
+ this.ioFailureTolerance = 0;
}
}
@@ -220,6 +251,8 @@ public abstract class StorageVolume
this.tmpDir =
new File(new File(getStorageDir(), workDirName), TMP_DIR_NAME);
Files.createDirectories(tmpDir.toPath());
+ diskCheckDir = createTmpSubdirIfNeeded(TMP_DISK_CHECK_DIR_NAME);
+ cleanTmpDiskCheckDir();
}
/**
@@ -438,6 +471,11 @@ public abstract class StorageVolume
return this.tmpDir;
}
+ @VisibleForTesting
+ public File getDiskCheckDir() {
+ return this.diskCheckDir;
+ }
+
public void refreshVolumeInfo() {
volumeInfo.ifPresent(VolumeInfo::refreshNow);
}
@@ -509,21 +547,122 @@ public abstract class StorageVolume
public void shutdown() {
setState(VolumeState.NON_EXISTENT);
volumeInfo.ifPresent(VolumeInfo::shutdownUsageThread);
+ cleanTmpDiskCheckDir();
+ }
+
+ /**
+ * Delete all temporary files in the directory used ot check disk health.
+ */
+ private void cleanTmpDiskCheckDir() {
+ // If the volume was shut down before initialization completed, skip
+ // emptying the directory.
+ if (diskCheckDir == null) {
+ return;
+ }
+
+ if (!diskCheckDir.exists()) {
+ LOG.warn("Unable to clear disk check files from {}. Directory does " +
+ "not exist.", diskCheckDir);
+ return;
+ }
+
+ if (!diskCheckDir.isDirectory()) {
+ LOG.warn("Unable to clear disk check files from {}. Location is not a" +
+ " directory", diskCheckDir);
+ return;
+ }
+
+ try (Stream<Path> files = Files.list(diskCheckDir.toPath())) {
+ files.map(Path::toFile).filter(File::isFile).forEach(file -> {
+ try {
+ Files.delete(file.toPath());
+ } catch (IOException ex) {
+ LOG.warn("Failed to delete temporary volume health check file {}",
+ file);
+ }
+ });
+ } catch (IOException ex) {
+ LOG.warn("Failed to list contents of volume health check directory {} " +
+ "for deleting.", diskCheckDir);
+ }
}
/**
- * Run a check on the current volume to determine if it is healthy.
+ * Run a check on the current volume to determine if it is healthy. The
+ * check consists of a directory check and an IO check.
+ *
+ * If the directory check fails, the volume check fails immediately.
+ * The IO check is allows to fail up to {@code ioFailureTolerance} times
+ * out of the last {@code ioTestCount} IO checks before this volume check is
+ * failed. Each call to this method runs one IO check.
+ *
* @param unused context for the check, ignored.
* @return result of checking the volume.
+ * @throws InterruptedException if there was an error during the volume
+ * check because the thread was interrupted.
* @throws Exception if an exception was encountered while running
- * the volume check.
+ * the volume check and the thread was not interrupted.
*/
@Override
- public VolumeCheckResult check(@Nullable Boolean unused) throws Exception {
- if (!storageDir.exists()) {
+ public synchronized VolumeCheckResult check(@Nullable Boolean unused)
+ throws Exception {
+ boolean directoryChecksPassed =
+ DiskCheckUtil.checkExistence(storageDir) &&
+ DiskCheckUtil.checkPermissions(storageDir);
+ // If the directory is not present or has incorrect permissions, fail the
+ // volume immediately. This is not an intermittent error.
+ if (!directoryChecksPassed) {
+ if (Thread.currentThread().isInterrupted()) {
+ throw new InterruptedException("Directory check of volume " + this +
+ " interrupted.");
+ }
+ return VolumeCheckResult.FAILED;
+ }
+
+ // If IO test count is set to 0, IO tests for disk health are disabled.
+ if (ioTestCount == 0) {
+ return VolumeCheckResult.HEALTHY;
+ }
+
+ // Since IO errors may be intermittent, volume remains healthy until the
+ // threshold of failures is crossed.
+ boolean diskChecksPassed = DiskCheckUtil.checkReadWrite(storageDir,
+ diskCheckDir, healthCheckFileSize);
+ if (Thread.currentThread().isInterrupted()) {
+ // Thread interrupt may have caused IO operations to abort. Do not
+ // consider this a failure.
+ throw new InterruptedException("IO check of volume " + this +
+ " interrupted.");
+ }
+
+ // Move the sliding window of IO test results forward 1 by adding the
+ // latest entry and removing the oldest entry from the window.
+ // Update the failure counter for the new window.
+ ioTestSlidingWindow.add(diskChecksPassed);
+ if (!diskChecksPassed) {
+ currentIOFailureCount.incrementAndGet();
+ }
+ if (ioTestSlidingWindow.size() > ioTestCount &&
+ Objects.equals(ioTestSlidingWindow.poll(), Boolean.FALSE)) {
+ currentIOFailureCount.decrementAndGet();
+ }
+
+ // If the failure threshold has been crossed, fail the volume without
+ // further scans.
+ // Once the volume is failed, it will not be checked anymore.
+ // The failure counts can be left as is.
+ if (currentIOFailureCount.get() > ioFailureTolerance) {
+ LOG.info("Failed IO test for volume {}: the last {} runs " +
+ "encountered {}/{} tolerated failures.", this,
+ ioTestSlidingWindow.size(), currentIOFailureCount,
+ ioFailureTolerance);
return VolumeCheckResult.FAILED;
+ } else if (LOG.isDebugEnabled()) {
+ LOG.debug("IO test results for volume {}: the last {} runs encountered "
+
+ "{}/{} tolerated failures", this, ioTestSlidingWindow.size(),
+ currentIOFailureCount, ioFailureTolerance);
}
- DiskChecker.checkDir(storageDir);
+
return VolumeCheckResult.HEALTHY;
}
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/StorageVolumeChecker.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/StorageVolumeChecker.java
index fe61a10d59..d9869894b2 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/StorageVolumeChecker.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/StorageVolumeChecker.java
@@ -318,6 +318,7 @@ public class StorageVolumeChecker {
switch (result) {
case HEALTHY:
case DEGRADED:
+ // Ozone does not currently use this state.
if (LOG.isDebugEnabled()) {
LOG.debug("Volume {} is {}.", volume, result);
}
@@ -343,8 +344,12 @@ public class StorageVolumeChecker {
t.getCause() : t;
LOG.warn("Exception running disk checks against volume {}",
volume, exception);
- markFailed();
- cleanup();
+ // If the scan was interrupted, do not count it as a volume failure.
+ // This should only happen if the volume checker is being shut down.
+ if (!(t instanceof InterruptedException)) {
+ markFailed();
+ cleanup();
+ }
}
private void markHealthy() {
diff --git
a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/utils/TestDiskCheckUtil.java
b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/utils/TestDiskCheckUtil.java
new file mode 100644
index 0000000000..701d13d81f
--- /dev/null
+++
b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/utils/TestDiskCheckUtil.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.container.common.utils;
+
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import java.io.File;
+
+/**
+ * Tests {@link DiskCheckUtil} does not incorrectly identify an unhealthy
+ * disk or mount point.
+ * Tests that it identifies an improperly configured directory mount point.
+ *
+ */
+public class TestDiskCheckUtil {
+ @Rule
+ public TemporaryFolder tempTestDir = new TemporaryFolder();
+
+ private File testDir;
+
+ @Before
+ public void setup() {
+ testDir = tempTestDir.getRoot();
+ }
+
+ @Test
+ public void testPermissions() {
+ // Ensure correct test setup before testing the disk check.
+ Assert.assertTrue(testDir.canRead());
+ Assert.assertTrue(testDir.canWrite());
+ Assert.assertTrue(testDir.canExecute());
+ Assert.assertTrue(DiskCheckUtil.checkPermissions(testDir));
+
+ // Test failure without read permissiosns.
+ Assert.assertTrue(testDir.setReadable(false));
+ Assert.assertFalse(DiskCheckUtil.checkPermissions(testDir));
+ Assert.assertTrue(testDir.setReadable(true));
+
+ // Test failure without write permissiosns.
+ Assert.assertTrue(testDir.setWritable(false));
+ Assert.assertFalse(DiskCheckUtil.checkPermissions(testDir));
+ Assert.assertTrue(testDir.setWritable(true));
+
+ // Test failure without execute permissiosns.
+ Assert.assertTrue(testDir.setExecutable(false));
+ Assert.assertFalse(DiskCheckUtil.checkPermissions(testDir));
+ Assert.assertTrue(testDir.setExecutable(true));
+ }
+
+ @Test
+ public void testExistence() {
+ // Ensure correct test setup before testing the disk check.
+ Assert.assertTrue(testDir.exists());
+ Assert.assertTrue(DiskCheckUtil.checkExistence(testDir));
+
+ Assert.assertTrue(testDir.delete());
+ Assert.assertFalse(DiskCheckUtil.checkExistence(testDir));
+ }
+
+ @Test
+ public void testReadWrite() {
+ Assert.assertTrue(DiskCheckUtil.checkReadWrite(testDir, testDir, 10));
+
+ // Test file should have been deleted.
+ File[] children = testDir.listFiles();
+ Assert.assertNotNull(children);
+ Assert.assertEquals(0, children.length);
+ }
+}
diff --git
a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestHddsVolume.java
b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestHddsVolume.java
index 53da489a3a..d02b5733d5 100644
---
a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestHddsVolume.java
+++
b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestHddsVolume.java
@@ -24,6 +24,7 @@ import java.time.Duration;
import java.util.UUID;
import java.util.concurrent.atomic.AtomicLong;
+import org.apache.commons.io.FileUtils;
import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.conf.StorageSize;
@@ -32,6 +33,7 @@ import org.apache.hadoop.hdds.fs.SpaceUsageCheckFactory;
import org.apache.hadoop.hdds.fs.SpaceUsagePersistence;
import org.apache.hadoop.hdds.fs.SpaceUsageSource;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.hdfs.server.datanode.checker.VolumeCheckResult;
import org.apache.hadoop.ozone.OzoneConfigKeys;
import static org.apache.hadoop.hdds.fs.MockSpaceUsagePersistence.inMemory;
@@ -119,26 +121,29 @@ public class TestHddsVolume {
// All temp directories should have been created.
assertTrue(volume.getTmpDir().exists());
assertTrue(volume.getDeletedContainerDir().exists());
+ assertTrue(volume.getDiskCheckDir().exists());
volume.shutdown();
// tmp directories should still exist after shutdown. This is not
// checking their contents.
assertTrue(volume.getTmpDir().exists());
assertTrue(volume.getDeletedContainerDir().exists());
+ assertTrue(volume.getDiskCheckDir().exists());
}
@Test
- public void testClearVolumeTmpDirs() throws Exception {
+ public void testClearDeletedContainersDir() throws Exception {
// Set up volume.
HddsVolume volume = volumeBuilder.build();
volume.format(CLUSTER_ID);
File tmpDir = volume.getHddsRootDir().toPath()
.resolve(Paths.get(CLUSTER_ID, StorageVolume.TMP_DIR_NAME)).toFile();
- File tmpDeleteDir = new File(tmpDir,
- HddsVolume.TMP_CONTAINER_DELETE_DIR_NAME);
+
// Simulate a container that failed to delete fully from the deleted
// containers directory.
+ File tmpDeleteDir = new File(tmpDir,
+ HddsVolume.TMP_CONTAINER_DELETE_DIR_NAME);
File leftoverContainer = new File(tmpDeleteDir, "1");
assertTrue(leftoverContainer.mkdirs());
@@ -162,6 +167,42 @@ public class TestHddsVolume {
assertTrue(tmpDeleteDir.exists());
}
+ @Test
+ public void testClearVolumeHealthCheckDir() throws Exception {
+ // Set up volume.
+ HddsVolume volume = volumeBuilder.build();
+ volume.format(CLUSTER_ID);
+
+ File tmpDir = volume.getHddsRootDir().toPath()
+ .resolve(Paths.get(CLUSTER_ID, StorageVolume.TMP_DIR_NAME)).toFile();
+
+ // Simulate a leftover disk check file that failed to delete.
+ File tmpDiskCheckDir = new File(tmpDir,
+ StorageVolume.TMP_DISK_CHECK_DIR_NAME);
+ assertTrue(tmpDiskCheckDir.mkdirs());
+ File leftoverDiskCheckFile = new File(tmpDiskCheckDir, "diskcheck");
+ assertTrue(leftoverDiskCheckFile.createNewFile());
+
+ // Check that tmp dirs are created with expected names.
+ volume.createWorkingDir(CLUSTER_ID, null);
+ volume.createTmpDirs(CLUSTER_ID);
+ assertEquals(tmpDir, volume.getTmpDir());
+ assertEquals(tmpDiskCheckDir, volume.getDiskCheckDir());
+
+ // Cleanup should have removed the leftover disk check file without
+ // removing the directory itself.
+ assertFalse(leftoverDiskCheckFile.exists());
+ assertTrue(tmpDiskCheckDir.exists());
+
+ // Re-create the disk check file
+ assertTrue(leftoverDiskCheckFile.createNewFile());
+
+ volume.shutdown();
+ // It should be cleared again on shutdown.
+ assertFalse(leftoverDiskCheckFile.exists());
+ assertTrue(tmpDiskCheckDir.exists());
+ }
+
@Test
public void testShutdown() throws Exception {
long initialUsedSpace = 250;
@@ -446,6 +487,25 @@ public class TestHddsVolume {
}
}
+ @Test
+ public void testDBDirFailureDetected() throws Exception {
+ HddsVolume volume = volumeBuilder.build();
+ volume.format(CLUSTER_ID);
+ volume.createWorkingDir(CLUSTER_ID, null);
+ volume.createTmpDirs(CLUSTER_ID);
+
+ VolumeCheckResult result = volume.check(false);
+ assertEquals(VolumeCheckResult.HEALTHY, result);
+
+ File dbFile = new File(volume.getDbParentDir(), CONTAINER_DB_NAME);
+ FileUtils.deleteDirectory(dbFile);
+
+ result = volume.check(false);
+ assertEquals(VolumeCheckResult.FAILED, result);
+
+ volume.shutdown();
+ }
+
private MutableVolumeSet createDbVolumeSet() throws IOException {
File dbVolumeDir = folder.newFolder();
CONF.set(OzoneConfigKeys.HDDS_DATANODE_CONTAINER_DB_DIR,
diff --git
a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestStorageVolume.java
b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestStorageVolume.java
index 5f015204fa..74469c78b5 100644
---
a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestStorageVolume.java
+++
b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestStorageVolume.java
@@ -16,9 +16,12 @@
*/
package org.apache.hadoop.ozone.container.common.volume;
+import org.apache.hadoop.hdfs.server.datanode.checker.VolumeCheckResult;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.fs.MockSpaceUsageCheckFactory;
import org.apache.hadoop.ozone.container.common.helpers.DatanodeVersionFile;
+import
org.apache.hadoop.ozone.container.common.statemachine.DatanodeConfiguration;
+import org.apache.hadoop.ozone.container.common.utils.DiskCheckUtil;
import org.apache.hadoop.ozone.container.common.utils.StorageVolumeUtil;
import org.junit.Before;
import org.junit.Rule;
@@ -30,6 +33,7 @@ import java.util.Properties;
import java.util.UUID;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
/**
* Test for StorageVolume.
@@ -46,6 +50,15 @@ public class TestStorageVolume {
private HddsVolume.Builder volumeBuilder;
private File versionFile;
+ private static final DiskCheckUtil.DiskChecks IO_FAILURE =
+ new DiskCheckUtil.DiskChecks() {
+ @Override
+ public boolean checkReadWrite(File storageDir, File testFileDir,
+ int numBytesToWrite) {
+ return false;
+ }
+ };
+
@Before
public void setup() throws Exception {
File rootDir = new File(folder.getRoot(), HddsVolume.HDDS_VOLUME_DIR);
@@ -54,6 +67,7 @@ public class TestStorageVolume {
.conf(CONF)
.usageCheckFactory(MockSpaceUsageCheckFactory.NONE);
versionFile = StorageVolumeUtil.getVersionFile(rootDir);
+ DiskCheckUtil.clearTestImpl();
}
@Test
@@ -80,4 +94,175 @@ public class TestStorageVolume {
assertEquals(volume.getCTime(), cTime);
assertEquals(volume.getLayoutVersion(), layoutVersion);
}
+
+ @Test
+ public void testCheckExistence() throws Exception {
+ HddsVolume volume = volumeBuilder.build();
+ volume.format(CLUSTER_ID);
+
+ VolumeCheckResult result = volume.check(false);
+ assertEquals(VolumeCheckResult.HEALTHY, result);
+
+ final DiskCheckUtil.DiskChecks doesNotExist =
+ new DiskCheckUtil.DiskChecks() {
+ @Override
+ public boolean checkExistence(File storageDir) {
+ return false;
+ }
+ };
+
+ DiskCheckUtil.setTestImpl(doesNotExist);
+ result = volume.check(false);
+ assertEquals(VolumeCheckResult.FAILED, result);
+ }
+
+ @Test
+ public void testCheckPermissions() throws Exception {
+ HddsVolume volume = volumeBuilder.build();
+ volume.format(CLUSTER_ID);
+
+ VolumeCheckResult result = volume.check(false);
+ assertEquals(VolumeCheckResult.HEALTHY, result);
+
+ final DiskCheckUtil.DiskChecks noPermissions =
+ new DiskCheckUtil.DiskChecks() {
+ @Override
+ public boolean checkPermissions(File storageDir) {
+ return false;
+ }
+ };
+
+ DiskCheckUtil.setTestImpl(noPermissions);
+ result = volume.check(false);
+ assertEquals(VolumeCheckResult.FAILED, result);
+ }
+
+ /**
+ * Setting test count to 0 should disable IO tests.
+ */
+ @Test
+ public void testCheckIODisabled() throws Exception {
+ DatanodeConfiguration dnConf = CONF.getObject(DatanodeConfiguration.class);
+ dnConf.setVolumeIOTestCount(0);
+ CONF.setFromObject(dnConf);
+ volumeBuilder.conf(CONF);
+ HddsVolume volume = volumeBuilder.build();
+ volume.format(CLUSTER_ID);
+
+ DiskCheckUtil.setTestImpl(IO_FAILURE);
+ assertEquals(VolumeCheckResult.HEALTHY, volume.check(false));
+ }
+
+ @Test
+ public void testCheckIODefaultConfigs() {
+ CONF.clear();
+ DatanodeConfiguration dnConf = CONF.getObject(DatanodeConfiguration.class);
+ // Make sure default values are not invalid.
+ assertTrue(dnConf.getVolumeIOFailureTolerance() <
+ dnConf.getVolumeIOTestCount());
+ }
+
+ @Test
+ public void testCheckIOInvalidConfig() throws Exception {
+ HddsVolume volume = volumeBuilder.build();
+ volume.format(CLUSTER_ID);
+ DatanodeConfiguration dnConf = CONF.getObject(DatanodeConfiguration.class);
+
+ // When failure tolerance is above test count, default values should be
+ // used.
+ dnConf.setVolumeIOTestCount(3);
+ dnConf.setVolumeIOFailureTolerance(4);
+ CONF.setFromObject(dnConf);
+ dnConf = CONF.getObject(DatanodeConfiguration.class);
+ assertEquals(dnConf.getVolumeIOTestCount(),
+ DatanodeConfiguration.DISK_CHECK_IO_TEST_COUNT_DEFAULT);
+ assertEquals(dnConf.getVolumeIOFailureTolerance(),
+ DatanodeConfiguration.DISK_CHECK_IO_FAILURES_TOLERATED_DEFAULT);
+
+ // When test count and failure tolerance are set to the same value,
+ // Default values should be used.
+ dnConf.setVolumeIOTestCount(2);
+ dnConf.setVolumeIOFailureTolerance(2);
+ CONF.setFromObject(dnConf);
+ dnConf = CONF.getObject(DatanodeConfiguration.class);
+ assertEquals(DatanodeConfiguration.DISK_CHECK_IO_TEST_COUNT_DEFAULT,
+ dnConf.getVolumeIOTestCount());
+
assertEquals(DatanodeConfiguration.DISK_CHECK_IO_FAILURES_TOLERATED_DEFAULT,
+ dnConf.getVolumeIOFailureTolerance());
+
+ // Negative test count should reset to default value.
+ dnConf.setVolumeIOTestCount(-1);
+ CONF.setFromObject(dnConf);
+ dnConf = CONF.getObject(DatanodeConfiguration .class);
+ assertEquals(DatanodeConfiguration.DISK_CHECK_IO_TEST_COUNT_DEFAULT,
+ dnConf.getVolumeIOTestCount());
+
+ // Negative failure tolerance should reset to default value.
+ dnConf.setVolumeIOFailureTolerance(-1);
+ CONF.setFromObject(dnConf);
+ dnConf = CONF.getObject(DatanodeConfiguration .class);
+
assertEquals(DatanodeConfiguration.DISK_CHECK_IO_FAILURES_TOLERATED_DEFAULT,
+ dnConf.getVolumeIOFailureTolerance());
+ }
+
+ @Test
+ public void testCheckIOInitiallyPassing() throws Exception {
+ testCheckIOUntilFailure(3, 1, true, true, true, false, true, false);
+ }
+
+ @Test
+ public void testCheckIOEarlyFailure() throws Exception {
+ testCheckIOUntilFailure(3, 1, false, false);
+ }
+
+ @Test
+ public void testCheckIOFailuresDiscarded() throws Exception {
+ testCheckIOUntilFailure(3, 1, false, true, true, true, false, false);
+ }
+
+ @Test
+ public void testCheckIOAlternatingFailures() throws Exception {
+ testCheckIOUntilFailure(3, 1, true, false, true, false);
+ }
+
+ /**
+ * Helper method to test the sliding window of IO checks before volume
+ * failure.
+ *
+ * @param ioTestCount The number of most recent tests whose results should
+ * be considered.
+ * @param ioFailureTolerance The number of IO failures tolerated out of the
+ * last {@param ioTestCount} tests.
+ * @param checkResults The result of the IO check for each run. Volume
+ * should fail after the last IO check is completed.
+ */
+ private void testCheckIOUntilFailure(int ioTestCount, int ioFailureTolerance,
+ boolean... checkResults) throws Exception {
+ DatanodeConfiguration dnConf = CONF.getObject(DatanodeConfiguration.class);
+ dnConf.setVolumeIOTestCount(ioTestCount);
+ dnConf.setVolumeIOFailureTolerance(ioFailureTolerance);
+ CONF.setFromObject(dnConf);
+ volumeBuilder.conf(CONF);
+ HddsVolume volume = volumeBuilder.build();
+ volume.format(CLUSTER_ID);
+
+ for (int i = 0; i < checkResults.length; i++) {
+ final boolean result = checkResults[i];
+ final DiskCheckUtil.DiskChecks ioResult = new DiskCheckUtil.DiskChecks()
{
+ @Override
+ public boolean checkReadWrite(File storageDir, File testDir,
+ int numBytesToWrite) {
+ return result;
+ }
+ };
+ DiskCheckUtil.setTestImpl(ioResult);
+ if (i < checkResults.length - 1) {
+ assertEquals("Unexpected IO failure in run " + i,
+ VolumeCheckResult.HEALTHY, volume.check(false));
+ } else {
+ assertEquals("Unexpected IO success in run " + i,
+ VolumeCheckResult.FAILED, volume.check(false));
+ }
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
