This is an automated email from the ASF dual-hosted git repository.
ayushsaxena pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 9eb7a8b HDFS-15041. Make MAX_LOCK_HOLD_MS and full queue size
configurable. Contributed by zhuqi.
9eb7a8b is described below
commit 9eb7a8bdf8f3b1dc76efc22db9651474303d309e
Author: Ayush Saxena <[email protected]>
AuthorDate: Sun Feb 23 14:25:39 2020 +0530
HDFS-15041. Make MAX_LOCK_HOLD_MS and full queue size configurable.
Contributed by zhuqi.
---
.../src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java | 4 ++++
.../hadoop/hdfs/server/blockmanagement/BlockManager.java | 10 ++++++++--
.../hadoop-hdfs/src/main/resources/hdfs-default.xml | 8 ++++++++
3 files changed, 20 insertions(+), 2 deletions(-)
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index f0f8f3e..459b9f8 100755
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -290,6 +290,10 @@ public class DFSConfigKeys extends CommonConfigurationKeys
{
= "dfs.namenode.blockreport.queue.size";
public static final int DFS_NAMENODE_BLOCKREPORT_QUEUE_SIZE_DEFAULT
= 1024;
+ public static final String DFS_NAMENODE_BLOCKREPORT_MAX_LOCK_HOLD_TIME
+ = "dfs.namenode.blockreport.max.lock.hold.time";
+ public static final long
+ DFS_NAMENODE_BLOCKREPORT_MAX_LOCK_HOLD_TIME_DEFAULT = 4;
@Deprecated
public static final String DFS_WEBHDFS_USER_PATTERN_KEY =
HdfsClientConfigKeys.DFS_WEBHDFS_USER_PATTERN_KEY;
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
index 45d3b79..aaa4aeb 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
@@ -393,6 +393,9 @@ public class BlockManager implements BlockStatsMXBean {
// Max number of blocks to log info about during a block report.
private final long maxNumBlocksToLog;
+ // Max write lock hold time for BlockReportProcessingThread(ms).
+ private final long maxLockHoldTime;
+
/**
* When running inside a Standby node, the node may receive block reports
* from datanodes before receiving the corresponding namespace edits from
@@ -543,6 +546,10 @@ public class BlockManager implements BlockStatsMXBean {
this.maxNumBlocksToLog =
conf.getLong(DFSConfigKeys.DFS_MAX_NUM_BLOCKS_TO_LOG_KEY,
DFSConfigKeys.DFS_MAX_NUM_BLOCKS_TO_LOG_DEFAULT);
+ this.maxLockHoldTime = conf.getTimeDuration(
+ DFSConfigKeys.DFS_NAMENODE_BLOCKREPORT_MAX_LOCK_HOLD_TIME,
+ DFSConfigKeys.DFS_NAMENODE_BLOCKREPORT_MAX_LOCK_HOLD_TIME_DEFAULT,
+ TimeUnit.MILLISECONDS);
this.numBlocksPerIteration = conf.getInt(
DFSConfigKeys.DFS_BLOCK_MISREPLICATION_PROCESSING_LIMIT,
DFSConfigKeys.DFS_BLOCK_MISREPLICATION_PROCESSING_LIMIT_DEFAULT);
@@ -5159,7 +5166,6 @@ public class BlockManager implements BlockStatsMXBean {
}
private class BlockReportProcessingThread extends Thread {
- private static final long MAX_LOCK_HOLD_MS = 4;
private long lastFull = 0;
private final BlockingQueue<Runnable> queue;
@@ -5195,7 +5201,7 @@ public class BlockManager implements BlockStatsMXBean {
do {
processed++;
action.run();
- if (Time.monotonicNow() - start > MAX_LOCK_HOLD_MS) {
+ if (Time.monotonicNow() - start > maxLockHoldTime) {
break;
}
action = queue.poll();
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
index 5daaffb..abb8dec 100755
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
@@ -5737,6 +5737,14 @@
</property>
<property>
+ <name>dfs.namenode.blockreport.max.lock.hold.time</name>
+ <value>4</value>
+ <description>
+ The BlockReportProcessingThread max write lock hold time in ms.
+ </description>
+ </property>
+
+ <property>
<name>dfs.journalnode.edits.dir.perm</name>
<value>700</value>
<description>
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
