This is an automated email from the ASF dual-hosted git repository. dineshc pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push: new c3134ab HDFS-15814. Make some parameters configurable for DataNodeDiskMetrics (#2676) c3134ab is described below commit c3134ab3a99d4109d9ae3fcf216018351eb1d36f Author: litao <55134131+toms...@users.noreply.github.com> AuthorDate: Sun Feb 14 13:12:24 2021 +0800 HDFS-15814. Make some parameters configurable for DataNodeDiskMetrics (#2676) --- .../java/org/apache/hadoop/hdfs/DFSConfigKeys.java | 8 +++++++ .../hadoop/hdfs/server/datanode/DataNode.java | 2 +- .../datanode/metrics/DataNodeDiskMetrics.java | 26 +++++++++++++++++----- .../src/main/resources/hdfs-default.xml | 16 +++++++++++++ 4 files changed, 46 insertions(+), 6 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java index bf0ed14..d04a842 100755 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java @@ -676,6 +676,14 @@ public class DFSConfigKeys extends CommonConfigurationKeys { "dfs.datanode.slowpeer.low.threshold.ms"; public static final long DFS_DATANODE_SLOWPEER_LOW_THRESHOLD_MS_DEFAULT = 5L; + public static final String DFS_DATANODE_MIN_OUTLIER_DETECTION_DISKS_KEY = + "dfs.datanode.min.outlier.detection.disks"; + public static final long DFS_DATANODE_MIN_OUTLIER_DETECTION_DISKS_DEFAULT = + 5L; + public static final String DFS_DATANODE_SLOWDISK_LOW_THRESHOLD_MS_KEY = + "dfs.datanode.slowdisk.low.threshold.ms"; + public static final long DFS_DATANODE_SLOWDISK_LOW_THRESHOLD_MS_DEFAULT = + 20L; public static final String DFS_DATANODE_HOST_NAME_KEY = HdfsClientConfigKeys.DeprecatedKeys.DFS_DATANODE_HOST_NAME_KEY; public static final String DFS_NAMENODE_CHECKPOINT_DIR_KEY = diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java index b16fe44..742b815 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java @@ -1489,7 +1489,7 @@ public class DataNode extends ReconfigurableBase if (dnConf.diskStatsEnabled) { diskMetrics = new DataNodeDiskMetrics(this, - dnConf.outliersReportIntervalMs); + dnConf.outliersReportIntervalMs, getConf()); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeDiskMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeDiskMetrics.java index e431bde..d0969f7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeDiskMetrics.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeDiskMetrics.java @@ -17,6 +17,8 @@ */ package org.apache.hadoop.hdfs.server.datanode.metrics; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableMap; import org.apache.hadoop.thirdparty.com.google.common.collect.Maps; @@ -48,8 +50,6 @@ public class DataNodeDiskMetrics { DataNodeDiskMetrics.class); private DataNode dn; - private final long MIN_OUTLIER_DETECTION_DISKS = 5; - private final long SLOW_DISK_LOW_THRESHOLD_MS = 20; private final long detectionInterval; private volatile boolean shouldRun; private OutlierDetector slowDiskDetector; @@ -61,11 +61,27 @@ public class DataNodeDiskMetrics { // code, status should not be overridden by daemon thread. private boolean overrideStatus = true; - public DataNodeDiskMetrics(DataNode dn, long diskOutlierDetectionIntervalMs) { + /** + * Minimum number of disks to run outlier detection. + */ + private final long minOutlierDetectionDisks; + /** + * Threshold in milliseconds below which a disk is definitely not slow. + */ + private final long lowThresholdMs; + + public DataNodeDiskMetrics(DataNode dn, long diskOutlierDetectionIntervalMs, + Configuration conf) { this.dn = dn; this.detectionInterval = diskOutlierDetectionIntervalMs; - slowDiskDetector = new OutlierDetector(MIN_OUTLIER_DETECTION_DISKS, - SLOW_DISK_LOW_THRESHOLD_MS); + minOutlierDetectionDisks = + conf.getLong(DFSConfigKeys.DFS_DATANODE_MIN_OUTLIER_DETECTION_DISKS_KEY, + DFSConfigKeys.DFS_DATANODE_MIN_OUTLIER_DETECTION_DISKS_DEFAULT); + lowThresholdMs = + conf.getLong(DFSConfigKeys.DFS_DATANODE_SLOWDISK_LOW_THRESHOLD_MS_KEY, + DFSConfigKeys.DFS_DATANODE_SLOWDISK_LOW_THRESHOLD_MS_DEFAULT); + slowDiskDetector = + new OutlierDetector(minOutlierDetectionDisks, lowThresholdMs); shouldRun = true; startDiskOutlierDetectionThread(); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml index e13e000..0c5bb35 100755 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml @@ -2371,6 +2371,22 @@ </property> <property> + <name>dfs.datanode.min.outlier.detection.disks</name> + <value>5</value> + <description> + Minimum number of disks to run outlier detection. + </description> +</property> + +<property> + <name>dfs.datanode.slowdisk.low.threshold.ms</name> + <value>20</value> + <description> + Threshold in milliseconds below which a disk is definitely not slow. + </description> +</property> + +<property> <name>hadoop.user.group.metrics.percentiles.intervals</name> <value></value> <description> --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org