Repository: hadoop Updated Branches: refs/heads/trunk 0424056a7 -> 22be604fa
Revert "HDFS-10534. NameNode WebUI should display DataNode usage rate with a certain percentile. Contributed by Kai Sasaki." This reverts commit 0424056a77002f4a2334ee2eb240fbc67b676471. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/22be604f Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/22be604f Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/22be604f Branch: refs/heads/trunk Commit: 22be604fa3e4054a8f329296be7334c1531b874f Parents: 0424056 Author: Zhe Zhang <[email protected]> Authored: Fri Jun 24 09:36:03 2016 -0700 Committer: Zhe Zhang <[email protected]> Committed: Fri Jun 24 09:36:03 2016 -0700 ---------------------------------------------------------------------- .../org/apache/hadoop/hdfs/DFSConfigKeys.java | 5 ----- .../hdfs/server/namenode/FSNamesystem.java | 21 +------------------- .../src/main/resources/hdfs-default.xml | 11 ---------- .../src/main/webapps/hdfs/dfshealth.html | 4 ++-- .../server/namenode/TestNameNodeMXBean.java | 10 ---------- 5 files changed, 3 insertions(+), 48 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/22be604f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java index 9371f56..6640ec6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java @@ -451,11 +451,6 @@ public class DFSConfigKeys extends CommonConfigurationKeys { "dfs.namenode.metrics.logger.period.seconds"; public static final int DFS_NAMENODE_METRICS_LOGGER_PERIOD_SECONDS_DEFAULT = 600; - public static final String DFS_NAMENODE_METRICS_NODE_USAGE_PERCENTILE = - "dfs.namenode.metrics.node-usage.percentile"; - public static final double DFS_NAMENODE_METRICS_NODE_USAGE_PERCENTILE_DEFAULT - = 0.95; - public static final String DFS_DATANODE_METRICS_LOGGER_PERIOD_SECONDS_KEY = "dfs.datanode.metrics.logger.period.seconds"; public static final int DFS_DATANODE_METRICS_LOGGER_PERIOD_SECONDS_DEFAULT = http://git-wip-us.apache.org/repos/asf/hadoop/blob/22be604f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index 02aaeda..915ae97 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -66,8 +66,6 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_ENABLE_RETRY_CAC import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_INODE_ATTRIBUTES_PROVIDER_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_LAZY_PERSIST_FILE_SCRUB_INTERVAL_SEC; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_LAZY_PERSIST_FILE_SCRUB_INTERVAL_SEC_DEFAULT; -import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_METRICS_NODE_USAGE_PERCENTILE; -import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_METRICS_NODE_USAGE_PERCENTILE_DEFAULT; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_MAX_OBJECTS_DEFAULT; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_MAX_OBJECTS_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY; @@ -530,8 +528,6 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, private INodeAttributeProvider inodeAttributeProvider; - private final double percentileFactor; - /** * If the NN is in safemode, and not due to manual / low resources, we * assume it must be because of startup. If the NN had low resources during @@ -828,15 +824,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, alwaysUseDelegationTokensForTests = conf.getBoolean( DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_KEY, DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_DEFAULT); - - this.percentileFactor = conf.getDouble( - DFS_NAMENODE_METRICS_NODE_USAGE_PERCENTILE, - DFS_NAMENODE_METRICS_NODE_USAGE_PERCENTILE_DEFAULT); - - Preconditions.checkArgument(0.0 < this.percentileFactor - && this.percentileFactor <= 1.0, "Node usage percentile " + - "factor must be between 0 and 1."); - + this.dtSecretManager = createDelegationTokenSecretManager(conf); this.dir = new FSDirectory(this, conf); this.snapshotManager = new SnapshotManager(dir); @@ -5626,7 +5614,6 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, float max = 0; float min = 0; float dev = 0; - float percentile = 0; final Map<String, Map<String,Object>> info = new HashMap<String, Map<String,Object>>(); @@ -5652,7 +5639,6 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, median = usages[usages.length / 2]; max = usages[usages.length - 1]; min = usages[0]; - percentile = usages[(int)((usages.length - 1) * percentileFactor)]; for (i = 0; i < usages.length; i++) { dev += (usages[i] - totalDfsUsed) * (usages[i] - totalDfsUsed); @@ -5665,11 +5651,6 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, innerInfo.put("median", StringUtils.format("%.2f%%", median)); innerInfo.put("max", StringUtils.format("%.2f%%", max)); innerInfo.put("stdDev", StringUtils.format("%.2f%%", dev)); - final Map<String, Object> percentileInfo = new HashMap<String, Object>(); - percentileInfo.put("name", StringUtils.format("%dth percentile", - (int)(percentileFactor * 100))); - percentileInfo.put("value", StringUtils.format("%.2f%%", percentile)); - innerInfo.put("percentile", percentileInfo); info.put("nodeUsage", innerInfo); return JSON.toString(info); http://git-wip-us.apache.org/repos/asf/hadoop/blob/22be604f/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml index bc56dc8..856e6b4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml @@ -1897,17 +1897,6 @@ </property> <property> - <name>dfs.namenode.metrics.node-usage.percentile</name> - <value>0.95</value> - <description> - This setting specifies the percentile level to report node usage metrics. - For example 0.95 means reporting the node usage for the 95th percentile - of all DataNodes. If this setting is at 0.95 and the reported node usage - is 70%, it means 95% of DataNodes have a usage below 70%. - </description> -</property> - -<property> <name>dfs.datanode.metrics.logger.period.seconds</name> <value>600</value> <description> http://git-wip-us.apache.org/repos/asf/hadoop/blob/22be604f/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html index d9a1f07..4fa2e4c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html @@ -166,8 +166,8 @@ <tr><th> Non DFS Used:</th><td>{NonDfsUsedSpace|fmt_bytes}</td></tr> <tr><th> DFS Remaining:</th><td>{Free|fmt_bytes} ({PercentRemaining|fmt_percentage})</td></tr> <tr><th> Block Pool Used:</th><td>{BlockPoolUsedSpace|fmt_bytes} ({PercentBlockPoolUsed|fmt_percentage})</td></tr> - <tr><th> {#NodeUsage.nodeUsage}DataNodes usages% (Min/Median/Max/stdDev/{percentile.name}):{/NodeUsage.nodeUsage} </th> - <td>{#NodeUsage.nodeUsage}{min} / {median} / {max} / {stdDev} / {percentile.value}{/NodeUsage.nodeUsage}</td></tr> + <tr><th> DataNodes usages% (Min/Median/Max/stdDev): </th> + <td>{#NodeUsage.nodeUsage}{min} / {median} / {max} / {stdDev}{/NodeUsage.nodeUsage}</td></tr> {/nn} {#fs} http://git-wip-us.apache.org/repos/asf/hadoop/blob/22be604f/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java index 3807076..dc8bea7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java @@ -173,16 +173,6 @@ public class TestNameNodeMXBean { String nodeUsage = (String) (mbs.getAttribute(mxbeanName, "NodeUsage")); assertEquals("Bad value for NodeUsage", fsn.getNodeUsage(), nodeUsage); - Map<String, Map<String, Object>> usage - = (Map<String, Map<String, Object>>)JSON.parse(nodeUsage); - assertTrue(usage.get("nodeUsage").containsKey("min")); - assertTrue(usage.get("nodeUsage").containsKey("median")); - assertTrue(usage.get("nodeUsage").containsKey("max")); - assertTrue(usage.get("nodeUsage").containsKey("percentile")); - Map<String, Object> percentileInfo - = (Map<String, Object>)usage.get("nodeUsage").get("percentile"); - assertTrue(percentileInfo.containsKey("name")); - assertTrue(percentileInfo.containsKey("value")); // get attribute NameJournalStatus String nameJournalStatus = (String) (mbs.getAttribute(mxbeanName, "NameJournalStatus")); --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
