This is an automated email from the ASF dual-hosted git repository.
tasanuma pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.3 by this push:
new 5601d08 HDFS-15854. Make some parameters configurable for
SlowDiskTracker and SlowPeerTracker (#2718)
5601d08 is described below
commit 5601d0848a5b6216eefc394c521f55c8a15ef94c
Author: litao <[email protected]>
AuthorDate: Mon Mar 1 23:52:59 2021 +0800
HDFS-15854. Make some parameters configurable for SlowDiskTracker and
SlowPeerTracker (#2718)
Authored-by: tomscut <[email protected]>
(cherry picked from commit 32353eb38a5bc73ad5ed75cfb446679d4e74958a)
---
.../java/org/apache/hadoop/hdfs/DFSConfigKeys.java | 8 ++++++++
.../hdfs/server/blockmanagement/SlowDiskTracker.java | 7 +++++--
.../hdfs/server/blockmanagement/SlowPeerTracker.java | 7 +++++--
.../hadoop-hdfs/src/main/resources/hdfs-default.xml | 18 ++++++++++++++++++
4 files changed, 36 insertions(+), 4 deletions(-)
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index dc2bcbe..505e64b 100755
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -658,6 +658,10 @@ public class DFSConfigKeys extends CommonConfigurationKeys
{
"dfs.datanode.slowpeer.low.threshold.ms";
public static final long DFS_DATANODE_SLOWPEER_LOW_THRESHOLD_MS_DEFAULT =
5L;
+ public static final String DFS_DATANODE_MAX_NODES_TO_REPORT_KEY =
+ "dfs.datanode.max.nodes.to.report";
+ public static final int DFS_DATANODE_MAX_NODES_TO_REPORT_DEFAULT =
+ 5;
public static final String DFS_DATANODE_MIN_OUTLIER_DETECTION_DISKS_KEY =
"dfs.datanode.min.outlier.detection.disks";
public static final long DFS_DATANODE_MIN_OUTLIER_DETECTION_DISKS_DEFAULT =
@@ -666,6 +670,10 @@ public class DFSConfigKeys extends CommonConfigurationKeys
{
"dfs.datanode.slowdisk.low.threshold.ms";
public static final long DFS_DATANODE_SLOWDISK_LOW_THRESHOLD_MS_DEFAULT =
20L;
+ public static final String DFS_DATANODE_MAX_DISKS_TO_REPORT_KEY =
+ "dfs.datanode.max.disks.to.report";
+ public static final int DFS_DATANODE_MAX_DISKS_TO_REPORT_DEFAULT =
+ 5;
public static final String DFS_DATANODE_HOST_NAME_KEY =
HdfsClientConfigKeys.DeprecatedKeys.DFS_DATANODE_HOST_NAME_KEY;
public static final String DFS_NAMENODE_CHECKPOINT_DIR_KEY =
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SlowDiskTracker.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SlowDiskTracker.java
index 08ebf8e..7823401 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SlowDiskTracker.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SlowDiskTracker.java
@@ -77,7 +77,7 @@ public class SlowDiskTracker {
* Number of disks to include in JSON report per operation. We will return
* disks with the highest latency.
*/
- private static final int MAX_DISKS_TO_REPORT = 5;
+ private final int maxDisksToReport;
private static final String DATANODE_DISK_SEPARATOR = ":";
private final long reportGenerationIntervalMs;
@@ -107,6 +107,9 @@ public class SlowDiskTracker {
DFSConfigKeys.DFS_DATANODE_OUTLIERS_REPORT_INTERVAL_KEY,
DFSConfigKeys.DFS_DATANODE_OUTLIERS_REPORT_INTERVAL_DEFAULT,
TimeUnit.MILLISECONDS);
+ this.maxDisksToReport = conf.getInt(
+ DFSConfigKeys.DFS_DATANODE_MAX_DISKS_TO_REPORT_KEY,
+ DFSConfigKeys.DFS_DATANODE_MAX_DISKS_TO_REPORT_DEFAULT);
this.reportValidityMs = reportGenerationIntervalMs * 3;
}
@@ -153,7 +156,7 @@ public class SlowDiskTracker {
@Override
public void run() {
slowDisksReport = getSlowDisks(diskIDLatencyMap,
- MAX_DISKS_TO_REPORT, now);
+ maxDisksToReport, now);
cleanUpOldReports(now);
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SlowPeerTracker.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SlowPeerTracker.java
index 453818e..c4b1861 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SlowPeerTracker.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SlowPeerTracker.java
@@ -80,7 +80,7 @@ public class SlowPeerTracker {
* Number of nodes to include in JSON report. We will return nodes with
* the highest number of votes from peers.
*/
- private static final int MAX_NODES_TO_REPORT = 5;
+ private final int maxNodesToReport;
/**
* Information about peers that have reported a node as being slow.
@@ -104,6 +104,9 @@ public class SlowPeerTracker {
DFSConfigKeys.DFS_DATANODE_OUTLIERS_REPORT_INTERVAL_KEY,
DFSConfigKeys.DFS_DATANODE_OUTLIERS_REPORT_INTERVAL_DEFAULT,
TimeUnit.MILLISECONDS) * 3;
+ this.maxNodesToReport = conf.getInt(
+ DFSConfigKeys.DFS_DATANODE_MAX_NODES_TO_REPORT_KEY,
+ DFSConfigKeys.DFS_DATANODE_MAX_NODES_TO_REPORT_DEFAULT);
}
/**
@@ -194,7 +197,7 @@ public class SlowPeerTracker {
*/
public String getJson() {
Collection<ReportForJson> validReports = getJsonReports(
- MAX_NODES_TO_REPORT);
+ maxNodesToReport);
try {
return WRITER.writeValueAsString(validReports);
} catch (JsonProcessingException e) {
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
index eee5f93..ea146a6 100755
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
@@ -2347,6 +2347,15 @@
</property>
<property>
+ <name>dfs.datanode.max.nodes.to.report</name>
+ <value>5</value>
+ <description>
+ Number of nodes to include in JSON report. We will return nodes with
+ the highest number of votes from peers.
+ </description>
+</property>
+
+<property>
<name>dfs.datanode.outliers.report.interval</name>
<value>30m</value>
<description>
@@ -2417,6 +2426,15 @@
</property>
<property>
+ <name>dfs.datanode.max.disks.to.report</name>
+ <value>5</value>
+ <description>
+ Number of disks to include in JSON report per operation. We will return
+ disks with the highest latency.
+ </description>
+</property>
+
+<property>
<name>hadoop.user.group.metrics.percentiles.intervals</name>
<value></value>
<description>
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
