HDFS-9086. Rename dfs.datanode.stripedread.threshold.millis to dfs.datanode.stripedread.timeout.millis. Contributed by Andrew Wang.
Change-Id: Ice86c5d46d29e94087c0f186b941d5394e7ac0e6 Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/82a88b92 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/82a88b92 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/82a88b92 Branch: refs/heads/HDFS-7240 Commit: 82a88b92b46911c1b33d11898f8f678a134a9b69 Parents: 607bda2 Author: Zhe Zhang <zhezh...@cloudera.com> Authored: Fri Sep 18 10:45:29 2015 -0700 Committer: Zhe Zhang <zhezh...@cloudera.com> Committed: Fri Sep 18 10:45:29 2015 -0700 ---------------------------------------------------------------------- hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt | 3 +++ .../main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java | 4 ++-- .../server/datanode/erasurecode/ErasureCodingWorker.java | 10 +++++----- .../org/apache/hadoop/hdfs/util/StripedBlockUtil.java | 6 +++--- .../hadoop-hdfs/src/main/resources/hdfs-default.xml | 8 ++++---- .../hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md | 2 +- 6 files changed, 18 insertions(+), 15 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/82a88b92/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt index 3a29db1..8ff696b 100755 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt @@ -432,3 +432,6 @@ HDFS-9097. Erasure coding: update EC command "-s" flag to "-p" when specifying policy. (zhz) + + HDFS-9086. Rename dfs.datanode.stripedread.threshold.millis to + dfs.datanode.stripedread.timeout.millis. (wang via zhz) http://git-wip-us.apache.org/repos/asf/hadoop/blob/82a88b92/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java index f7cda18..53c6cdb 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java @@ -399,8 +399,8 @@ public class DFSConfigKeys extends CommonConfigurationKeys { public static final int DFS_DATANODE_STRIPED_READ_THREADS_DEFAULT = 20; public static final String DFS_DATANODE_STRIPED_READ_BUFFER_SIZE_KEY = "dfs.datanode.stripedread.buffer.size"; public static final int DFS_DATANODE_STRIPED_READ_BUFFER_SIZE_DEFAULT = 64 * 1024; - public static final String DFS_DATANODE_STRIPED_READ_THRESHOLD_MILLIS_KEY = "dfs.datanode.stripedread.threshold.millis"; - public static final int DFS_DATANODE_STRIPED_READ_THRESHOLD_MILLIS_DEFAULT = 5000; //5s + public static final String DFS_DATANODE_STRIPED_READ_TIMEOUT_MILLIS_KEY = "dfs.datanode.stripedread.timeout.millis"; + public static final int DFS_DATANODE_STRIPED_READ_TIMEOUT_MILLIS_DEFAULT = 5000; //5s public static final String DFS_DATANODE_STRIPED_BLK_RECOVERY_THREADS_KEY = "dfs.datanode.striped.blockrecovery.threads.size"; public static final int DFS_DATANODE_STRIPED_BLK_RECOVERY_THREADS_DEFAULT = 8; public static final String DFS_DATANODE_DNS_INTERFACE_KEY = "dfs.datanode.dns.interface"; http://git-wip-us.apache.org/repos/asf/hadoop/blob/82a88b92/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/ErasureCodingWorker.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/ErasureCodingWorker.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/ErasureCodingWorker.java index 56b54f1..d1fa478 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/ErasureCodingWorker.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/ErasureCodingWorker.java @@ -95,16 +95,16 @@ public final class ErasureCodingWorker { private ThreadPoolExecutor STRIPED_BLK_RECOVERY_THREAD_POOL; private ThreadPoolExecutor STRIPED_READ_THREAD_POOL; - private final int STRIPED_READ_THRESHOLD_MILLIS; + private final int STRIPED_READ_TIMEOUT_MILLIS; private final int STRIPED_READ_BUFFER_SIZE; public ErasureCodingWorker(Configuration conf, DataNode datanode) { this.datanode = datanode; this.conf = conf; - STRIPED_READ_THRESHOLD_MILLIS = conf.getInt( - DFSConfigKeys.DFS_DATANODE_STRIPED_READ_THRESHOLD_MILLIS_KEY, - DFSConfigKeys.DFS_DATANODE_STRIPED_READ_THRESHOLD_MILLIS_DEFAULT); + STRIPED_READ_TIMEOUT_MILLIS = conf.getInt( + DFSConfigKeys.DFS_DATANODE_STRIPED_READ_TIMEOUT_MILLIS_KEY, + DFSConfigKeys.DFS_DATANODE_STRIPED_READ_TIMEOUT_MILLIS_DEFAULT); initializeStripedReadThreadPool(conf.getInt( DFSConfigKeys.DFS_DATANODE_STRIPED_READ_THREADS_KEY, DFSConfigKeys.DFS_DATANODE_STRIPED_READ_THREADS_DEFAULT)); @@ -556,7 +556,7 @@ public final class ErasureCodingWorker { try { StripingChunkReadResult result = StripedBlockUtil.getNextCompletedStripedRead( - readService, futures, STRIPED_READ_THRESHOLD_MILLIS); + readService, futures, STRIPED_READ_TIMEOUT_MILLIS); int resultIndex = -1; if (result.state == StripingChunkReadResult.SUCCESSFUL) { resultIndex = result.index; http://git-wip-us.apache.org/repos/asf/hadoop/blob/82a88b92/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/StripedBlockUtil.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/StripedBlockUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/StripedBlockUtil.java index 4fded73..897b092 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/StripedBlockUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/StripedBlockUtil.java @@ -203,12 +203,12 @@ public class StripedBlockUtil { */ public static StripingChunkReadResult getNextCompletedStripedRead( CompletionService<Void> readService, Map<Future<Void>, Integer> futures, - final long threshold) throws InterruptedException { + final long timeoutMillis) throws InterruptedException { Preconditions.checkArgument(!futures.isEmpty()); Future<Void> future = null; try { - if (threshold > 0) { - future = readService.poll(threshold, TimeUnit.MILLISECONDS); + if (timeoutMillis > 0) { + future = readService.poll(timeoutMillis, TimeUnit.MILLISECONDS); } else { future = readService.take(); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/82a88b92/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml index 0db56dd..acbad07 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml @@ -2403,23 +2403,23 @@ </property> <property> - <name>dfs.datanode.stripedread.threshold.millis</name> + <name>dfs.datanode.stripedread.timeout.millis</name> <value>5000</value> - <description>datanode striped read threshold in millisecond. + <description>Datanode striped read timeout in milliseconds. </description> </property> <property> <name>dfs.datanode.stripedread.threads</name> <value>20</value> - <description>datanode striped read thread pool size. + <description>Number of threads used by the Datanode for background recovery work. </description> </property> <property> <name>dfs.datanode.stripedread.buffer.size</name> <value>262144</value> - <description>datanode striped read buffer size. + <description>Datanode striped read buffer size. </description> </property> http://git-wip-us.apache.org/repos/asf/hadoop/blob/82a88b92/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md index 2323a6d..5b3aa34 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md +++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md @@ -101,7 +101,7 @@ Deployment Erasure coding background recovery work on the DataNodes can also be tuned via the following configuration parameters: - 1. `dfs.datanode.stripedread.threshold.millis` - Timeout for striped reads. Default value is 5000 ms. + 1. `dfs.datanode.stripedread.timeout.millis` - Timeout for striped reads. Default value is 5000 ms. 1. `dfs.datanode.stripedread.threads` - Number of concurrent reader threads. Default value is 20 threads. 1. `dfs.datanode.stripedread.buffer.size` - Buffer size for reader service. Default value is 256KB.