Repository: hadoop Updated Branches: refs/heads/branch-2 b6c477691 -> 5cdcda34a
HDFS-11477. Simplify file IO profiling configuration. Contributed by Hanisha Koneru. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5cdcda34 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5cdcda34 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5cdcda34 Branch: refs/heads/branch-2 Commit: 5cdcda34a2e859862eb68bff5844c18ac4d27917 Parents: b6c4776 Author: Arpit Agarwal <a...@apache.org> Authored: Thu Mar 9 14:34:10 2017 -0800 Committer: Arpit Agarwal <a...@apache.org> Committed: Thu Mar 9 14:34:10 2017 -0800 ---------------------------------------------------------------------- .../hadoop-common/src/site/markdown/Metrics.md | 7 +++++- .../org/apache/hadoop/hdfs/DFSConfigKeys.java | 8 ++---- .../apache/hadoop/hdfs/server/common/Util.java | 26 +++++++++++++++++--- .../hadoop/hdfs/server/datanode/DNConf.java | 7 +++--- .../hdfs/server/datanode/FileIoProvider.java | 2 +- .../server/datanode/ProfilingFileIoEvents.java | 11 ++++----- .../src/main/resources/hdfs-default.xml | 2 +- .../datanode/TestDataNodeVolumeMetrics.java | 4 +-- .../hadoop/tools/TestHdfsConfigFields.java | 2 -- 9 files changed, 43 insertions(+), 26 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/5cdcda34/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md b/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md index e52a6a7..020d1ad 100644 --- a/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md +++ b/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md @@ -323,7 +323,12 @@ Each metrics record contains tags such as SessionId and Hostname as additional i FsVolume -------- -Per-volume metrics contain Datanode Volume IO related statistics. Per-volume metrics are off by default. They can be enbabled by setting `dfs.datanode.enable.fileio.profiling` to **true**, but enabling per-volume metrics may have a performance impact. Each metrics record contains tags such as Hostname as additional information along with metrics. +Per-volume metrics contain Datanode Volume IO related statistics. Per-volume +metrics are off by default. They can be enabled by setting `dfs.datanode +.fileio.profiling.sampling.fraction` to a fraction between 0.0 and 1.0. +Setting this value to 0.0 would mean profiling is not enabled. But enabling +per-volume metrics may have a performance impact. Each metrics record +contains tags such as Hostname as additional information along with metrics. | Name | Description | |:---- |:---- | http://git-wip-us.apache.org/repos/asf/hadoop/blob/5cdcda34/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java index e9e6bcc..4732359 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java @@ -634,7 +634,7 @@ public class DFSConfigKeys extends CommonConfigurationKeys { public static final int DFS_BLOCK_MISREPLICATION_PROCESSING_LIMIT_DEFAULT = 10000; public static final String DFS_DATANODE_OUTLIERS_REPORT_INTERVAL_KEY = - "dfs.datanode.slow.peers.report.interval"; + "dfs.datanode.outliers.report.interval"; public static final int DFS_DATANODE_OUTLIERS_REPORT_INTERVAL_DEFAULT = 1800 * 1000; @@ -664,10 +664,6 @@ public class DFSConfigKeys extends CommonConfigurationKeys { public static final int DFS_IMAGE_TRANSFER_CHUNKSIZE_DEFAULT = 64 * 1024; // Datanode File IO Stats - public static final String DFS_DATANODE_ENABLE_FILEIO_PROFILING_KEY = - "dfs.datanode.enable.fileio.profiling"; - public static final boolean DFS_DATANODE_ENABLE_FILEIO_PROFILING_DEFAULT = - false; public static final String DFS_DATANODE_ENABLE_FILEIO_FAULT_INJECTION_KEY = "dfs.datanode.enable.fileio.fault.injection"; public static final boolean @@ -676,7 +672,7 @@ public class DFSConfigKeys extends CommonConfigurationKeys { DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_KEY = "dfs.datanode.fileio.profiling.sampling.fraction"; public static final double - DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_DEAFULT = 1.0; + DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_DEFAULT = 0.0; //Keys with no defaults public static final String DFS_DATANODE_PLUGINS_KEY = "dfs.datanode.plugins"; http://git-wip-us.apache.org/repos/asf/hadoop/blob/5cdcda34/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Util.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Util.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Util.java index 6ec686f..07ad118 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Util.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Util.java @@ -17,6 +17,11 @@ */ package org.apache.hadoop.hdfs.server.common; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.hdfs.DFSConfigKeys; + import java.io.File; import java.io.IOException; import java.net.URI; @@ -25,10 +30,6 @@ import java.util.ArrayList; import java.util.Collection; import java.util.List; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.classification.InterfaceAudience; - @InterfaceAudience.Private public final class Util { private final static Log LOG = LogFactory.getLog(Util.class.getName()); @@ -102,4 +103,21 @@ public final class Util { } return uris; } + + public static boolean isDiskStatsEnabled(double fileIOSamplingFraction) { + final boolean isEnabled; + if (fileIOSamplingFraction < 0.000001) { + LOG.info(DFSConfigKeys + .DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_KEY + " set to " + + fileIOSamplingFraction + ". Disabling file IO profiling"); + isEnabled = false; + } else { + LOG.info(DFSConfigKeys + .DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_KEY + " set to " + + fileIOSamplingFraction + ". Enabling file IO profiling"); + isEnabled = true; + } + + return isEnabled; + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/5cdcda34/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java index c98793c..de74681 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java @@ -63,6 +63,7 @@ import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys; import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.protocol.datatransfer.TrustedChannelResolver; import org.apache.hadoop.hdfs.protocol.datatransfer.sasl.DataTransferSaslUtil; +import org.apache.hadoop.hdfs.server.common.Util; import org.apache.hadoop.security.SaslPropertiesResolver; import java.util.concurrent.TimeUnit; @@ -171,9 +172,9 @@ public class DNConf { this.peerStatsEnabled = getConf().getBoolean( DFSConfigKeys.DFS_DATANODE_PEER_STATS_ENABLED_KEY, DFSConfigKeys.DFS_DATANODE_PEER_STATS_ENABLED_DEFAULT); - this.diskStatsEnabled = getConf().getBoolean( - DFSConfigKeys.DFS_DATANODE_ENABLE_FILEIO_PROFILING_KEY, - DFSConfigKeys.DFS_DATANODE_ENABLE_FILEIO_PROFILING_DEFAULT); + this.diskStatsEnabled = Util.isDiskStatsEnabled(getConf().getDouble( + DFSConfigKeys.DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_KEY, + DFSConfigKeys.DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_DEFAULT)); this.outliersReportIntervalMs = getConf().getTimeDuration( DFS_DATANODE_OUTLIERS_REPORT_INTERVAL_KEY, DFS_DATANODE_OUTLIERS_REPORT_INTERVAL_DEFAULT, http://git-wip-us.apache.org/repos/asf/hadoop/blob/5cdcda34/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/FileIoProvider.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/FileIoProvider.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/FileIoProvider.java index 108088d..5986b88 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/FileIoProvider.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/FileIoProvider.java @@ -62,7 +62,7 @@ import static org.apache.hadoop.hdfs.server.datanode.FileIoProvider.OPERATION.*; * * Behavior can be injected into these events by enabling the * profiling and/or fault injection event hooks through - * {@link DFSConfigKeys#DFS_DATANODE_ENABLE_FILEIO_PROFILING_KEY} and + * {@link DFSConfigKeys#DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_KEY} and * {@link DFSConfigKeys#DFS_DATANODE_ENABLE_FILEIO_FAULT_INJECTION_KEY}. * These event hooks are disabled by default. * http://git-wip-us.apache.org/repos/asf/hadoop/blob/5cdcda34/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ProfilingFileIoEvents.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ProfilingFileIoEvents.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ProfilingFileIoEvents.java index 61d0432..2990105 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ProfilingFileIoEvents.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ProfilingFileIoEvents.java @@ -23,6 +23,7 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.hdfs.server.common.Util; import org.apache.hadoop.hdfs.server.datanode.fsdataset.DataNodeVolumeMetrics; import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi; import org.apache.hadoop.util.Time; @@ -43,13 +44,11 @@ class ProfilingFileIoEvents { public ProfilingFileIoEvents(@Nullable Configuration conf) { if (conf != null) { - isEnabled = conf.getBoolean(DFSConfigKeys - .DFS_DATANODE_ENABLE_FILEIO_PROFILING_KEY, DFSConfigKeys - .DFS_DATANODE_ENABLE_FILEIO_PROFILING_DEFAULT); - double fileIOSamplingFraction = conf.getDouble(DFSConfigKeys - .DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_KEY, + double fileIOSamplingFraction = conf.getDouble( + DFSConfigKeys.DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_KEY, DFSConfigKeys - .DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_DEAFULT); + .DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_DEFAULT); + isEnabled = Util.isDiskStatsEnabled(fileIOSamplingFraction); if (fileIOSamplingFraction > 1) { LOG.warn(DFSConfigKeys .DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_KEY + http://git-wip-us.apache.org/repos/asf/hadoop/blob/5cdcda34/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml index ca08eb3..f9c691a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml @@ -1931,7 +1931,7 @@ <property> <name>dfs.datanode.outliers.report.interval</name> - <value>30m</value> + <value>1800000</value> <description> This setting controls how frequently DataNodes will report their peer latencies to the NameNode via heartbeats. This setting supports http://git-wip-us.apache.org/repos/asf/hadoop/blob/5cdcda34/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeMetrics.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeMetrics.java index 6a8ac9c..03e1dee 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeMetrics.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeMetrics.java @@ -121,8 +121,8 @@ public class TestDataNodeVolumeMetrics { private MiniDFSCluster setupClusterForVolumeMetrics() throws IOException { Configuration conf = new HdfsConfiguration(); - conf.setBoolean(DFSConfigKeys - .DFS_DATANODE_ENABLE_FILEIO_PROFILING_KEY, true); + conf.setDouble(DFSConfigKeys + .DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_KEY, 1.0); SimulatedFSDataset.setFactory(conf); return new MiniDFSCluster.Builder(conf) .numDataNodes(NUM_DATANODES) http://git-wip-us.apache.org/repos/asf/hadoop/blob/5cdcda34/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestHdfsConfigFields.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestHdfsConfigFields.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestHdfsConfigFields.java index da5ef4d..6b6c87a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestHdfsConfigFields.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestHdfsConfigFields.java @@ -107,8 +107,6 @@ public class TestHdfsConfigFields extends TestConfigurationFieldsBase { .add(DFSConfigKeys.DFS_DATANODE_STARTUP_KEY); configurationPropsToSkipCompare .add(DFSConfigKeys.DFS_NAMENODE_STARTUP_KEY); - configurationPropsToSkipCompare - .add(DFSConfigKeys.DFS_DATANODE_ENABLE_FILEIO_PROFILING_KEY); configurationPropsToSkipCompare.add(DFSConfigKeys .DFS_DATANODE_ENABLE_FILEIO_FAULT_INJECTION_KEY); configurationPropsToSkipCompare.add(DFSConfigKeys --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
