This is an automated email from the ASF dual-hosted git repository.
tasanuma pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.3 by this push:
new 4c57fb4 HDFS-15745. Make DataNodePeerMetrics#LOW_THRESHOLD_MS and
MIN_OUTLIER_DETECTION_NODES configurable. Contributed by Haibin Huang. (#3992)
4c57fb4 is described below
commit 4c57fb4d6bb4cfe9b9269dd07b09270a969c34cf
Author: Takanobu Asanuma <[email protected]>
AuthorDate: Wed Feb 16 09:42:43 2022 +0900
HDFS-15745. Make DataNodePeerMetrics#LOW_THRESHOLD_MS and
MIN_OUTLIER_DETECTION_NODES configurable. Contributed by Haibin Huang. (#3992)
(cherry picked from commit 1cd96e8dd81b220cc197011e80cd8e298c566e1a)
Conflicts:
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
Reviewed-by: Ayush Saxena <[email protected]>
---
.../java/org/apache/hadoop/hdfs/DFSConfigKeys.java | 8 ++++++++
.../datanode/metrics/DataNodePeerMetrics.java | 24 +++++++++++++++-------
.../src/main/resources/hdfs-default.xml | 16 +++++++++++++++
3 files changed, 41 insertions(+), 7 deletions(-)
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index 7196def..dc2bcbe 100755
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -650,6 +650,14 @@ public class DFSConfigKeys extends CommonConfigurationKeys
{
public static final long
DFS_DATANODE_PEER_METRICS_MIN_OUTLIER_DETECTION_SAMPLES_DEFAULT =
1000;
+ public static final String DFS_DATANODE_MIN_OUTLIER_DETECTION_NODES_KEY =
+ "dfs.datanode.min.outlier.detection.nodes";
+ public static final long DFS_DATANODE_MIN_OUTLIER_DETECTION_NODES_DEFAULT =
+ 10L;
+ public static final String DFS_DATANODE_SLOWPEER_LOW_THRESHOLD_MS_KEY =
+ "dfs.datanode.slowpeer.low.threshold.ms";
+ public static final long DFS_DATANODE_SLOWPEER_LOW_THRESHOLD_MS_DEFAULT =
+ 5L;
public static final String DFS_DATANODE_MIN_OUTLIER_DETECTION_DISKS_KEY =
"dfs.datanode.min.outlier.detection.disks";
public static final long DFS_DATANODE_MIN_OUTLIER_DETECTION_DISKS_DEFAULT =
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodePeerMetrics.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodePeerMetrics.java
index 3c70a23..750e53d 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodePeerMetrics.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodePeerMetrics.java
@@ -22,6 +22,7 @@ package org.apache.hadoop.hdfs.server.datanode.metrics;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.metrics2.MetricsJsonBuilder;
import org.apache.hadoop.metrics2.lib.MutableRollingAverages;
import org.slf4j.Logger;
@@ -48,11 +49,6 @@ public class DataNodePeerMetrics {
private final String name;
- /**
- * Threshold in milliseconds below which a DataNode is definitely not slow.
- */
- private static final long LOW_THRESHOLD_MS = 5;
- private static final long MIN_OUTLIER_DETECTION_NODES = 10;
private final OutlierDetector slowNodeDetector;
@@ -62,14 +58,28 @@ public class DataNodePeerMetrics {
* outlier detection is skipped.
*/
private final long minOutlierDetectionSamples;
+ /**
+ * Threshold in milliseconds below which a DataNode is definitely not slow.
+ */
+ private final long lowThresholdMs;
+ /**
+ * Minimum number of nodes to run outlier detection.
+ */
+ private final long minOutlierDetectionNodes;
public DataNodePeerMetrics(final String name, Configuration conf) {
this.name = name;
minOutlierDetectionSamples = conf.getLong(
DFS_DATANODE_PEER_METRICS_MIN_OUTLIER_DETECTION_SAMPLES_KEY,
DFS_DATANODE_PEER_METRICS_MIN_OUTLIER_DETECTION_SAMPLES_DEFAULT);
- this.slowNodeDetector = new OutlierDetector(MIN_OUTLIER_DETECTION_NODES,
- LOW_THRESHOLD_MS);
+ lowThresholdMs =
+ conf.getLong(DFSConfigKeys.DFS_DATANODE_SLOWPEER_LOW_THRESHOLD_MS_KEY,
+ DFSConfigKeys.DFS_DATANODE_SLOWPEER_LOW_THRESHOLD_MS_DEFAULT);
+ minOutlierDetectionNodes =
+
conf.getLong(DFSConfigKeys.DFS_DATANODE_MIN_OUTLIER_DETECTION_NODES_KEY,
+ DFSConfigKeys.DFS_DATANODE_MIN_OUTLIER_DETECTION_NODES_DEFAULT);
+ this.slowNodeDetector =
+ new OutlierDetector(minOutlierDetectionNodes, lowThresholdMs);
sendPacketDownstreamRollingAverages = new MutableRollingAverages("Time");
}
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
index af547fd..eee5f93 100755
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
@@ -2331,6 +2331,22 @@
</property>
<property>
+ <name>dfs.datanode.min.outlier.detection.nodes</name>
+ <value>10</value>
+ <description>
+ Minimum number of nodes to run outlier detection.
+ </description>
+</property>
+
+<property>
+ <name>dfs.datanode.slowpeer.low.threshold.ms</name>
+ <value>5</value>
+ <description>
+ Threshold in milliseconds below which a DataNode is definitely not slow.
+ </description>
+</property>
+
+<property>
<name>dfs.datanode.outliers.report.interval</name>
<value>30m</value>
<description>
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
