HDFS-12832. INode.getFullPathName may throw ArrayIndexOutOfBoundsException lead to NameNode exit. Contribuited by Konstantin Shvachko.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d331762f Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d331762f Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d331762f Branch: refs/heads/YARN-5881 Commit: d331762f24b3f22f609366740c9c4f449edc61ac Parents: 30941d9 Author: Konstantin V Shvachko <[email protected]> Authored: Tue Nov 28 17:14:23 2017 -0800 Committer: Konstantin V Shvachko <[email protected]> Committed: Tue Nov 28 17:14:23 2017 -0800 ---------------------------------------------------------------------- .../hdfs/server/blockmanagement/BlockManager.java | 2 -- .../blockmanagement/BlockReconstructionWork.java | 14 ++++++++++---- .../server/blockmanagement/ErasureCodingWork.java | 4 ++-- .../hdfs/server/blockmanagement/ReplicationWork.java | 4 ++-- 4 files changed, 14 insertions(+), 10 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/d331762f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java index bdabd81..4986027 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java @@ -1825,8 +1825,6 @@ public class BlockManager implements BlockStatsMXBean { } // choose replication targets: NOT HOLDING THE GLOBAL LOCK - // It is costly to extract the filename for which chooseTargets is called, - // so for now we pass in the block collection itself. final BlockPlacementPolicy placementPolicy = placementPolicies.getPolicy(rw.getBlock().getBlockType()); rw.chooseTargets(placementPolicy, storagePolicySuite, excludedNodes); http://git-wip-us.apache.org/repos/asf/hadoop/blob/d331762f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockReconstructionWork.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockReconstructionWork.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockReconstructionWork.java index 57121bd..3f591e8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockReconstructionWork.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockReconstructionWork.java @@ -32,7 +32,8 @@ import java.util.Set; abstract class BlockReconstructionWork { private final BlockInfo block; - private final BlockCollection bc; + private final String srcPath; + private final byte storagePolicyID; /** * An erasure coding reconstruction task has multiple source nodes. @@ -57,7 +58,8 @@ abstract class BlockReconstructionWork { int additionalReplRequired, int priority) { this.block = block; - this.bc = bc; + this.srcPath = bc.getName(); + this.storagePolicyID = bc.getStoragePolicyID(); this.srcNodes = srcNodes; this.containingNodes = containingNodes; this.liveReplicaStorages = liveReplicaStorages; @@ -94,8 +96,12 @@ abstract class BlockReconstructionWork { return srcNodes; } - BlockCollection getBc() { - return bc; + public String getSrcPath() { + return srcPath; + } + + public byte getStoragePolicyID() { + return storagePolicyID; } List<DatanodeStorageInfo> getLiveReplicaStorages() { http://git-wip-us.apache.org/repos/asf/hadoop/blob/d331762f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/ErasureCodingWork.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/ErasureCodingWork.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/ErasureCodingWork.java index 0ae6f0f..a23b1d5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/ErasureCodingWork.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/ErasureCodingWork.java @@ -58,10 +58,10 @@ class ErasureCodingWork extends BlockReconstructionWork { Set<Node> excludedNodes) { // TODO: new placement policy for EC considering multiple writers DatanodeStorageInfo[] chosenTargets = blockplacement.chooseTarget( - getBc().getName(), getAdditionalReplRequired(), getSrcNodes()[0], + getSrcPath(), getAdditionalReplRequired(), getSrcNodes()[0], getLiveReplicaStorages(), false, excludedNodes, getBlock().getNumBytes(), - storagePolicySuite.getPolicy(getBc().getStoragePolicyID()), null); + storagePolicySuite.getPolicy(getStoragePolicyID()), null); setTargets(chosenTargets); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/d331762f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/ReplicationWork.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/ReplicationWork.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/ReplicationWork.java index 6078b1d..26c38cb 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/ReplicationWork.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/ReplicationWork.java @@ -44,10 +44,10 @@ class ReplicationWork extends BlockReconstructionWork { : "At least 1 source node should have been selected"; try { DatanodeStorageInfo[] chosenTargets = blockplacement.chooseTarget( - getBc().getName(), getAdditionalReplRequired(), getSrcNodes()[0], + getSrcPath(), getAdditionalReplRequired(), getSrcNodes()[0], getLiveReplicaStorages(), false, excludedNodes, getBlock().getNumBytes(), - storagePolicySuite.getPolicy(getBc().getStoragePolicyID()), + storagePolicySuite.getPolicy(getStoragePolicyID()), null); setTargets(chosenTargets); } finally { --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
