Repository: hadoop Updated Branches: refs/heads/trunk 0ff121610 -> 73b86a504
HDFS-8988. Use LightWeightHashSet instead of LightWeightLinkedSet in BlockManager#excessReplicateMap. (yliu) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/73b86a50 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/73b86a50 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/73b86a50 Branch: refs/heads/trunk Commit: 73b86a5046fe3262dde7b05be46b18575e35fd5f Parents: 0ff1216 Author: yliu <[email protected]> Authored: Mon Oct 12 14:40:14 2015 +0800 Committer: yliu <[email protected]> Committed: Mon Oct 12 14:40:14 2015 +0800 ---------------------------------------------------------------------- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ .../server/blockmanagement/BlockManager.java | 23 ++++++++------------ .../hdfs/server/namenode/NamenodeFsck.java | 4 ++-- 3 files changed, 14 insertions(+), 16 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/73b86a50/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 4171433..938546c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -1512,6 +1512,9 @@ Release 2.8.0 - UNRELEASED HDFS-9221. HdfsServerConstants#ReplicaState#getState should avoid calling values() since it creates a temporary array. (Staffan Friberg via yliu) + HDFS-8988. Use LightWeightHashSet instead of LightWeightLinkedSet in + BlockManager#excessReplicateMap. (yliu) + OPTIMIZATIONS HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than http://git-wip-us.apache.org/repos/asf/hadoop/blob/73b86a50/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java index 18bfc41..8a64b74 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java @@ -89,7 +89,6 @@ import org.apache.hadoop.hdfs.server.protocol.KeyUpdateCommand; import org.apache.hadoop.hdfs.server.protocol.ReceivedDeletedBlockInfo; import org.apache.hadoop.hdfs.server.protocol.StorageReceivedDeletedBlocks; import org.apache.hadoop.hdfs.util.LightWeightHashSet; -import org.apache.hadoop.hdfs.util.LightWeightLinkedSet; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; import static org.apache.hadoop.hdfs.util.StripedBlockUtil.getInternalBlockLength; @@ -219,7 +218,7 @@ public class BlockManager implements BlockStatsMXBean { * Maps a StorageID to the set of blocks that are "extra" for this * DataNode. We'll eventually remove these extras. */ - public final Map<String, LightWeightLinkedSet<BlockInfo>> excessReplicateMap = + public final Map<String, LightWeightHashSet<BlockInfo>> excessReplicateMap = new HashMap<>(); /** @@ -1421,11 +1420,6 @@ public class BlockManager implements BlockStatsMXBean { */ @VisibleForTesting int computeRecoveryWorkForBlocks(List<List<BlockInfo>> blocksToRecover) { - int requiredReplication, numEffectiveReplicas; - List<DatanodeDescriptor> containingNodes; - BlockCollection bc; - int additionalReplRequired; - int scheduledWork = 0; List<BlockRecoveryWork> recovWork = new LinkedList<>(); @@ -1786,7 +1780,7 @@ public class BlockManager implements BlockStatsMXBean { Collection<DatanodeDescriptor> nodesCorrupt = corruptReplicas.getNodes(block); for (DatanodeStorageInfo storage : blocksMap.getStorages(block)) { final DatanodeDescriptor node = storage.getDatanodeDescriptor(); - LightWeightLinkedSet<BlockInfo> excessBlocks = + LightWeightHashSet<BlockInfo> excessBlocks = excessReplicateMap.get(node.getDatanodeUuid()); int countableReplica = storage.getState() == State.NORMAL ? 1 : 0; if ((nodesCorrupt != null) && (nodesCorrupt.contains(node))) @@ -3090,7 +3084,7 @@ public class BlockManager implements BlockStatsMXBean { postponeBlock(block); return; } - LightWeightLinkedSet<BlockInfo> excessBlocks = excessReplicateMap.get( + LightWeightHashSet<BlockInfo> excessBlocks = excessReplicateMap.get( cur.getDatanodeUuid()); if (excessBlocks == null || !excessBlocks.contains(block)) { if (!cur.isDecommissionInProgress() && !cur.isDecommissioned()) { @@ -3297,10 +3291,10 @@ public class BlockManager implements BlockStatsMXBean { private void addToExcessReplicate(DatanodeInfo dn, BlockInfo storedBlock) { assert namesystem.hasWriteLock(); - LightWeightLinkedSet<BlockInfo> excessBlocks = excessReplicateMap.get( + LightWeightHashSet<BlockInfo> excessBlocks = excessReplicateMap.get( dn.getDatanodeUuid()); if (excessBlocks == null) { - excessBlocks = new LightWeightLinkedSet<>(); + excessBlocks = new LightWeightHashSet<>(); excessReplicateMap.put(dn.getDatanodeUuid(), excessBlocks); } if (excessBlocks.add(storedBlock)) { @@ -3364,7 +3358,7 @@ public class BlockManager implements BlockStatsMXBean { // We've removed a block from a node, so it's definitely no longer // in "excess" there. // - LightWeightLinkedSet<BlockInfo> excessBlocks = excessReplicateMap.get( + LightWeightHashSet<BlockInfo> excessBlocks = excessReplicateMap.get( node.getDatanodeUuid()); if (excessBlocks != null) { if (excessBlocks.remove(storedBlock)) { @@ -3581,7 +3575,7 @@ public class BlockManager implements BlockStatsMXBean { } else if (node.isDecommissioned()) { decommissioned++; } else { - LightWeightLinkedSet<BlockInfo> blocksExcess = excessReplicateMap.get( + LightWeightHashSet<BlockInfo> blocksExcess = excessReplicateMap.get( node.getDatanodeUuid()); if (blocksExcess != null && blocksExcess.contains(b)) { excess++; @@ -3988,7 +3982,8 @@ public class BlockManager implements BlockStatsMXBean { private void removeFromExcessReplicateMap(Block block) { for (DatanodeStorageInfo info : blocksMap.getStorages(block)) { String uuid = info.getDatanodeDescriptor().getDatanodeUuid(); - LightWeightLinkedSet<BlockInfo> excessReplicas = excessReplicateMap.get(uuid); + LightWeightHashSet<BlockInfo> excessReplicas = + excessReplicateMap.get(uuid); if (excessReplicas != null) { if (excessReplicas.remove(block)) { excessBlocksCount.decrementAndGet(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/73b86a50/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java index 404a71e..2166639 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java @@ -74,7 +74,7 @@ import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor; import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeStorageInfo; import org.apache.hadoop.hdfs.server.blockmanagement.NumberReplicas; import org.apache.hadoop.hdfs.server.datanode.CachingStrategy; -import org.apache.hadoop.hdfs.util.LightWeightLinkedSet; +import org.apache.hadoop.hdfs.util.LightWeightHashSet; import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.net.NetworkTopology; import org.apache.hadoop.net.NodeBase; @@ -687,7 +687,7 @@ public class NamenodeFsck implements DataEncryptionKeyFactory { .getStorageType())); } if (showReplicaDetails) { - LightWeightLinkedSet<BlockInfo> blocksExcess = + LightWeightHashSet<BlockInfo> blocksExcess = bm.excessReplicateMap.get(dnDesc.getDatanodeUuid()); Collection<DatanodeDescriptor> corruptReplicas = bm.getCorruptReplicas(block.getLocalBlock());
