Repository: hadoop Updated Branches: refs/heads/trunk 4abb2fa68 -> f2ba7da4f
HDFS-9876. shouldProcessOverReplicated should not count number of pending replicas. Contributed by Jing Zhao. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f2ba7da4 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f2ba7da4 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f2ba7da4 Branch: refs/heads/trunk Commit: f2ba7da4f0df6cf0fc245093aeb4500158e6ee0b Parents: 4abb2fa Author: Jing Zhao <[email protected]> Authored: Tue Mar 1 18:41:57 2016 -0800 Committer: Jing Zhao <[email protected]> Committed: Tue Mar 1 18:41:57 2016 -0800 ---------------------------------------------------------------------- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++ .../server/blockmanagement/BlockManager.java | 38 ++++++++++---------- 2 files changed, 23 insertions(+), 18 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/f2ba7da4/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 80bd4d57..421ee68 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -455,6 +455,9 @@ Trunk (Unreleased) HDFS-9867. Missing block exception should carry locatedBlocks information. (Mingliang Liu via jing9) + HDFS-9876. shouldProcessOverReplicated should not count number of pending + replicas. (jing9) + BREAKDOWN OF HDFS-7285 SUBTASKS AND RELATED JIRAS HDFS-7347. Configurable erasure coding policy for individual files and http://git-wip-us.apache.org/repos/asf/hadoop/blob/f2ba7da4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java index 77eea0a..5175c13 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java @@ -2893,7 +2893,7 @@ public class BlockManager implements BlockStatsMXBean { } else { updateNeededReplications(storedBlock, curReplicaDelta, 0); } - if (shouldProcessOverReplicated(num, pendingNum, fileReplication)) { + if (shouldProcessOverReplicated(num, fileReplication)) { processOverReplicatedBlock(storedBlock, fileReplication, node, delNodeHint); } // If the file replication has reached desired value @@ -2912,8 +2912,8 @@ public class BlockManager implements BlockStatsMXBean { } private boolean shouldProcessOverReplicated(NumberReplicas num, - int pendingNum, int expectedNum) { - int numCurrent = num.liveReplicas() + pendingNum; + int expectedNum) { + final int numCurrent = num.liveReplicas(); return numCurrent > expectedNum || (numCurrent == expectedNum && num.redundantInternalBlocks() > 0); } @@ -3131,7 +3131,7 @@ public class BlockManager implements BlockStatsMXBean { } } - if (shouldProcessOverReplicated(num, 0, expectedReplication)) { + if (shouldProcessOverReplicated(num, expectedReplication)) { if (num.replicasOnStaleNodes() > 0) { // If any of the replicas of this block are on nodes that are // considered "stale", then these replicas may in fact have @@ -3268,7 +3268,6 @@ public class BlockManager implements BlockStatsMXBean { assert storedBlock instanceof BlockInfoStriped; BlockInfoStriped sblk = (BlockInfoStriped) storedBlock; short groupSize = sblk.getTotalBlockNum(); - BlockPlacementPolicy placementPolicy = placementPolicies.getPolicy(true); // find all duplicated indices BitSet found = new BitSet(groupSize); //indices found @@ -3283,14 +3282,6 @@ public class BlockManager implements BlockStatsMXBean { found.set(index); storage2index.put(storage, index); } - // the number of target left replicas equals to the of number of the found - // indices. - int numOfTarget = found.cardinality(); - - final BlockStoragePolicy storagePolicy = storagePolicySuite.getPolicy( - bc.getStoragePolicyID()); - final List<StorageType> excessTypes = storagePolicy.chooseExcess( - (short)numOfTarget, DatanodeStorageInfo.toStorageTypes(nonExcess)); // use delHint only if delHint is duplicated final DatanodeStorageInfo delStorageHint = @@ -3302,6 +3293,19 @@ public class BlockManager implements BlockStatsMXBean { } } + // cardinality of found indicates the expected number of internal blocks + final int numOfTarget = found.cardinality(); + final BlockStoragePolicy storagePolicy = storagePolicySuite.getPolicy( + bc.getStoragePolicyID()); + final List<StorageType> excessTypes = storagePolicy.chooseExcess( + (short) numOfTarget, DatanodeStorageInfo.toStorageTypes(nonExcess)); + if (excessTypes.isEmpty()) { + LOG.warn("excess types chosen for block {} among storages {} is empty", + storedBlock, nonExcess); + return; + } + + BlockPlacementPolicy placementPolicy = placementPolicies.getPolicy(true); // for each duplicated index, delete some replicas until only one left for (int targetIndex = duplicated.nextSetBit(0); targetIndex >= 0; targetIndex = duplicated.nextSetBit(targetIndex + 1)) { @@ -3312,9 +3316,7 @@ public class BlockManager implements BlockStatsMXBean { candidates.add(storage); } } - Block internalBlock = new Block(storedBlock); - internalBlock.setBlockId(storedBlock.getBlockId() + targetIndex); - while (candidates.size() > 1) { + if (candidates.size() > 1) { List<DatanodeStorageInfo> replicasToDelete = placementPolicy .chooseReplicasToDelete(nonExcess, candidates, (short) 1, excessTypes, null, null); @@ -3749,7 +3751,7 @@ public class BlockManager implements BlockStatsMXBean { final BlockInfo block = it.next(); int expectedReplication = this.getReplication(block); NumberReplicas num = countNodes(block); - if (shouldProcessOverReplicated(num, 0, expectedReplication)) { + if (shouldProcessOverReplicated(num, expectedReplication)) { // over-replicated block processOverReplicatedBlock(block, (short) expectedReplication, null, null); @@ -3886,7 +3888,7 @@ public class BlockManager implements BlockStatsMXBean { neededReplications.add(block, n.liveReplicas() + pending, n.readOnlyReplicas(), n.decommissionedAndDecommissioning(), expected); - } else if (shouldProcessOverReplicated(n, 0, expected)) { + } else if (shouldProcessOverReplicated(n, expected)) { processOverReplicatedBlock(block, expected, null, null); } }
