HDFS-10453. ReplicationMonitor thread could stuck for long time due to the race 
between replication and delete of same file in a large cluster.. Contributed by 
He Xiaoqiao.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/41cf2b8e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/41cf2b8e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/41cf2b8e

Branch: refs/heads/branch-3.0
Commit: 41cf2b8e7cf9f008272f513ab66edd252e999b80
Parents: 38febc0
Author: Arpit Agarwal <a...@apache.org>
Authored: Mon Feb 12 07:00:50 2018 -0800
Committer: Arpit Agarwal <a...@apache.org>
Committed: Mon Feb 12 07:16:48 2018 -0800

----------------------------------------------------------------------
 .../hdfs/server/blockmanagement/BlockReconstructionWork.java   | 6 ++++++
 .../hadoop/hdfs/server/blockmanagement/ErasureCodingWork.java  | 3 +--
 .../hadoop/hdfs/server/blockmanagement/ReplicationWork.java    | 6 ++----
 3 files changed, 9 insertions(+), 6 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/41cf2b8e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockReconstructionWork.java
----------------------------------------------------------------------
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockReconstructionWork.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockReconstructionWork.java
index 3f591e8..d383191 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockReconstructionWork.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockReconstructionWork.java
@@ -33,6 +33,7 @@ abstract class BlockReconstructionWork {
   private final BlockInfo block;
 
   private final String srcPath;
+  private final long blockSize;
   private final byte storagePolicyID;
 
   /**
@@ -59,6 +60,7 @@ abstract class BlockReconstructionWork {
       int priority) {
     this.block = block;
     this.srcPath = bc.getName();
+    this.blockSize = block.getNumBytes();
     this.storagePolicyID = bc.getStoragePolicyID();
     this.srcNodes = srcNodes;
     this.containingNodes = containingNodes;
@@ -100,6 +102,10 @@ abstract class BlockReconstructionWork {
     return srcPath;
   }
 
+  public long getBlockSize() {
+    return blockSize;
+  }
+
   public byte getStoragePolicyID() {
     return storagePolicyID;
   }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/41cf2b8e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/ErasureCodingWork.java
----------------------------------------------------------------------
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/ErasureCodingWork.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/ErasureCodingWork.java
index a23b1d5..147f8cf 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/ErasureCodingWork.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/ErasureCodingWork.java
@@ -59,8 +59,7 @@ class ErasureCodingWork extends BlockReconstructionWork {
     // TODO: new placement policy for EC considering multiple writers
     DatanodeStorageInfo[] chosenTargets = blockplacement.chooseTarget(
         getSrcPath(), getAdditionalReplRequired(), getSrcNodes()[0],
-        getLiveReplicaStorages(), false, excludedNodes,
-        getBlock().getNumBytes(),
+        getLiveReplicaStorages(), false, excludedNodes, getBlockSize(),
         storagePolicySuite.getPolicy(getStoragePolicyID()), null);
     setTargets(chosenTargets);
   }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/41cf2b8e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/ReplicationWork.java
----------------------------------------------------------------------
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/ReplicationWork.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/ReplicationWork.java
index 26c38cb..f250bcb 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/ReplicationWork.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/ReplicationWork.java
@@ -45,10 +45,8 @@ class ReplicationWork extends BlockReconstructionWork {
     try {
       DatanodeStorageInfo[] chosenTargets = blockplacement.chooseTarget(
           getSrcPath(), getAdditionalReplRequired(), getSrcNodes()[0],
-          getLiveReplicaStorages(), false, excludedNodes,
-          getBlock().getNumBytes(),
-          storagePolicySuite.getPolicy(getStoragePolicyID()),
-          null);
+          getLiveReplicaStorages(), false, excludedNodes, getBlockSize(),
+          storagePolicySuite.getPolicy(getStoragePolicyID()), null);
       setTargets(chosenTargets);
     } finally {
       getSrcNodes()[0].decrementPendingReplicationWithoutTargets();


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

Reply via email to