Repository: hadoop Updated Branches: refs/heads/trunk a853b4e1b -> 79e939d0b
HDFS-11339. Support File IO sampling for Datanode IO profiling hooks. Contributed by Hanisha Koneru. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/79e939d0 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/79e939d0 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/79e939d0 Branch: refs/heads/trunk Commit: 79e939d0b848a50200612c8c471db6bce1c822be Parents: a853b4e Author: Arpit Agarwal <a...@apache.org> Authored: Mon Jan 16 14:43:29 2017 -0800 Committer: Arpit Agarwal <a...@apache.org> Committed: Mon Jan 16 14:43:29 2017 -0800 ---------------------------------------------------------------------- .../org/apache/hadoop/hdfs/DFSConfigKeys.java | 16 +++++++++++---- .../server/datanode/ProfilingFileIoEvents.java | 21 ++++++++++++++++++-- .../hadoop/tools/TestHdfsConfigFields.java | 2 ++ 3 files changed, 33 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/79e939d0/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java index a00e872..8f60af0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java @@ -694,10 +694,7 @@ public class DFSConfigKeys extends CommonConfigurationKeys { public static final String DFS_IMAGE_TRANSFER_CHUNKSIZE_KEY = "dfs.image.transfer.chunksize"; public static final int DFS_IMAGE_TRANSFER_CHUNKSIZE_DEFAULT = 64 * 1024; - //Keys with no defaults - public static final String DFS_DATANODE_PLUGINS_KEY = "dfs.datanode.plugins"; - public static final String DFS_DATANODE_FSDATASET_FACTORY_KEY = "dfs.datanode.fsdataset.factory"; - public static final String DFS_DATANODE_FSDATASET_VOLUME_CHOOSING_POLICY_KEY = "dfs.datanode.fsdataset.volume.choosing.policy"; + // Datanode File IO Stats public static final String DFS_DATANODE_ENABLE_FILEIO_PROFILING_KEY = "dfs.datanode.enable.fileio.profiling"; public static final boolean DFS_DATANODE_ENABLE_FILEIO_PROFILING_DEFAULT = @@ -706,6 +703,17 @@ public class DFSConfigKeys extends CommonConfigurationKeys { "dfs.datanode.enable.fileio.fault.injection"; public static final boolean DFS_DATANODE_ENABLE_FILEIO_FAULT_INJECTION_DEFAULT = false; + public static final String + DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_KEY = + "dfs.datanode.fileio.profiling.sampling.fraction"; + public static final double + DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_DEAFULT = 1.0; + + //Keys with no defaults + public static final String DFS_DATANODE_PLUGINS_KEY = "dfs.datanode.plugins"; + public static final String DFS_DATANODE_FSDATASET_FACTORY_KEY = "dfs.datanode.fsdataset.factory"; + public static final String DFS_DATANODE_FSDATASET_VOLUME_CHOOSING_POLICY_KEY = "dfs.datanode.fsdataset.volume.choosing.policy"; + public static final String DFS_DATANODE_AVAILABLE_SPACE_VOLUME_CHOOSING_POLICY_BALANCED_SPACE_THRESHOLD_KEY = "dfs.datanode.available-space-volume-choosing-policy.balanced-space-threshold"; public static final long DFS_DATANODE_AVAILABLE_SPACE_VOLUME_CHOOSING_POLICY_BALANCED_SPACE_THRESHOLD_DEFAULT = 1024L * 1024L * 1024L * 10L; // 10 GB public static final String DFS_DATANODE_AVAILABLE_SPACE_VOLUME_CHOOSING_POLICY_BALANCED_SPACE_PREFERENCE_FRACTION_KEY = "dfs.datanode.available-space-volume-choosing-policy.balanced-space-preference-fraction"; http://git-wip-us.apache.org/repos/asf/hadoop/blob/79e939d0/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ProfilingFileIoEvents.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ProfilingFileIoEvents.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ProfilingFileIoEvents.java index 43ac495..6e689a6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ProfilingFileIoEvents.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ProfilingFileIoEvents.java @@ -18,12 +18,15 @@ package org.apache.hadoop.hdfs.server.datanode; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.server.datanode.fsdataset.DataNodeVolumeMetrics; import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi; import org.apache.hadoop.util.Time; +import org.jboss.netty.util.internal.ThreadLocalRandom; import javax.annotation.Nullable; @@ -33,16 +36,30 @@ import javax.annotation.Nullable; */ @InterfaceAudience.Private class ProfilingFileIoEvents { + static final Log LOG = LogFactory.getLog(ProfilingFileIoEvents.class); private final boolean isEnabled; + private final int sampleRangeMax; public ProfilingFileIoEvents(@Nullable Configuration conf) { if (conf != null) { isEnabled = conf.getBoolean(DFSConfigKeys .DFS_DATANODE_ENABLE_FILEIO_PROFILING_KEY, DFSConfigKeys .DFS_DATANODE_ENABLE_FILEIO_PROFILING_DEFAULT); + double fileIOSamplingFraction = conf.getDouble(DFSConfigKeys + .DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_KEY, + DFSConfigKeys + .DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_DEAFULT); + if (fileIOSamplingFraction > 1) { + LOG.warn(DFSConfigKeys + .DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_KEY + + " value cannot be more than 1. Setting value to 1"); + fileIOSamplingFraction = 1; + } + sampleRangeMax = (int) (fileIOSamplingFraction * Integer.MAX_VALUE); } else { isEnabled = false; + sampleRangeMax = 0; } } @@ -69,7 +86,7 @@ class ProfilingFileIoEvents { public long beforeFileIo(@Nullable FsVolumeSpi volume, FileIoProvider.OPERATION op, long len) { - if (isEnabled) { + if (isEnabled && ThreadLocalRandom.current().nextInt() < sampleRangeMax) { DataNodeVolumeMetrics metrics = getVolumeMetrics(volume); if (metrics != null) { return Time.monotonicNow(); @@ -80,7 +97,7 @@ class ProfilingFileIoEvents { public void afterFileIo(@Nullable FsVolumeSpi volume, FileIoProvider.OPERATION op, long begin, long len) { - if (isEnabled) { + if (isEnabled && begin != 0) { DataNodeVolumeMetrics metrics = getVolumeMetrics(volume); if (metrics != null) { long latency = Time.monotonicNow() - begin; http://git-wip-us.apache.org/repos/asf/hadoop/blob/79e939d0/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestHdfsConfigFields.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestHdfsConfigFields.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestHdfsConfigFields.java index 51ef5d0..6b0dced 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestHdfsConfigFields.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestHdfsConfigFields.java @@ -103,6 +103,8 @@ public class TestHdfsConfigFields extends TestConfigurationFieldsBase { .add(DFSConfigKeys.DFS_DATANODE_ENABLE_FILEIO_PROFILING_KEY); configurationPropsToSkipCompare.add(DFSConfigKeys .DFS_DATANODE_ENABLE_FILEIO_FAULT_INJECTION_KEY); + configurationPropsToSkipCompare.add(DFSConfigKeys + .DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_KEY); // Allocate xmlPropsToSkipCompare = new HashSet<String>(); --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
