HDFS-10453. ReplicationMonitor thread could stuck for long time due to the race between replication and delete of same file in a large cluster.. Contributed by He Xiaoqiao.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e82abfd6 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e82abfd6 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e82abfd6 Branch: refs/heads/branch-3.1 Commit: e82abfd64ba2f1f5dfb2a185a8388f8966149007 Parents: bd5ab59 Author: Arpit Agarwal <a...@apache.org> Authored: Mon Feb 12 07:00:50 2018 -0800 Committer: Arpit Agarwal <a...@apache.org> Committed: Mon Feb 12 07:17:50 2018 -0800 ---------------------------------------------------------------------- .../hdfs/server/blockmanagement/BlockReconstructionWork.java | 6 ++++++ .../hadoop/hdfs/server/blockmanagement/ErasureCodingWork.java | 3 +-- .../hadoop/hdfs/server/blockmanagement/ReplicationWork.java | 6 ++---- 3 files changed, 9 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/e82abfd6/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockReconstructionWork.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockReconstructionWork.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockReconstructionWork.java index 3f591e8..d383191 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockReconstructionWork.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockReconstructionWork.java @@ -33,6 +33,7 @@ abstract class BlockReconstructionWork { private final BlockInfo block; private final String srcPath; + private final long blockSize; private final byte storagePolicyID; /** @@ -59,6 +60,7 @@ abstract class BlockReconstructionWork { int priority) { this.block = block; this.srcPath = bc.getName(); + this.blockSize = block.getNumBytes(); this.storagePolicyID = bc.getStoragePolicyID(); this.srcNodes = srcNodes; this.containingNodes = containingNodes; @@ -100,6 +102,10 @@ abstract class BlockReconstructionWork { return srcPath; } + public long getBlockSize() { + return blockSize; + } + public byte getStoragePolicyID() { return storagePolicyID; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/e82abfd6/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/ErasureCodingWork.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/ErasureCodingWork.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/ErasureCodingWork.java index a23b1d5..147f8cf 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/ErasureCodingWork.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/ErasureCodingWork.java @@ -59,8 +59,7 @@ class ErasureCodingWork extends BlockReconstructionWork { // TODO: new placement policy for EC considering multiple writers DatanodeStorageInfo[] chosenTargets = blockplacement.chooseTarget( getSrcPath(), getAdditionalReplRequired(), getSrcNodes()[0], - getLiveReplicaStorages(), false, excludedNodes, - getBlock().getNumBytes(), + getLiveReplicaStorages(), false, excludedNodes, getBlockSize(), storagePolicySuite.getPolicy(getStoragePolicyID()), null); setTargets(chosenTargets); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/e82abfd6/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/ReplicationWork.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/ReplicationWork.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/ReplicationWork.java index 26c38cb..f250bcb 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/ReplicationWork.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/ReplicationWork.java @@ -45,10 +45,8 @@ class ReplicationWork extends BlockReconstructionWork { try { DatanodeStorageInfo[] chosenTargets = blockplacement.chooseTarget( getSrcPath(), getAdditionalReplRequired(), getSrcNodes()[0], - getLiveReplicaStorages(), false, excludedNodes, - getBlock().getNumBytes(), - storagePolicySuite.getPolicy(getStoragePolicyID()), - null); + getLiveReplicaStorages(), false, excludedNodes, getBlockSize(), + storagePolicySuite.getPolicy(getStoragePolicyID()), null); setTargets(chosenTargets); } finally { getSrcNodes()[0].decrementPendingReplicationWithoutTargets(); --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org