Repository: hadoop Updated Branches: refs/heads/branch-2.7 63e4ada51 -> 3e793224f
HDFS-6945. BlockManager should remove a block from excessReplicateMap and decrement ExcessBlocks metric when the block is removed. (aajisaka) (cherry picked from commit 18a91fe4df0448d9f7de91602646ecf5a51c52e4) (cherry picked from commit b85bbca74565b18dfa6689c9545d07bff5d31f83) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3e793224 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3e793224 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3e793224 Branch: refs/heads/branch-2.7 Commit: 3e793224f929bfc272dbfdb0f580208c8703d31f Parents: 63e4ada Author: Akira Ajisaka <aajis...@apache.org> Authored: Wed Apr 1 09:07:28 2015 +0900 Committer: Akira Ajisaka <aajis...@apache.org> Committed: Tue Jul 21 11:42:21 2015 +0900 ---------------------------------------------------------------------- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ .../server/blockmanagement/BlockManager.java | 22 ++++++++++++++++++-- .../namenode/metrics/TestNameNodeMetrics.java | 9 ++++++-- 3 files changed, 30 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/3e793224/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 5cfb9a5..8aab98a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -16,6 +16,9 @@ Release 2.7.2 - UNRELEASED BUG FIXES + HDFS-6945. BlockManager should remove a block from excessReplicateMap and + decrement ExcessBlocks metric when the block is removed. (aajisaka) + Release 2.7.1 - 2015-07-06 INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/3e793224/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java index 09e5748..d770346 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java @@ -3382,8 +3382,7 @@ public class BlockManager { // file already removes them from the block map below. block.setNumBytes(BlockCommand.NO_ACK); addToInvalidates(block); - corruptReplicas.removeFromCorruptReplicasMap(block); - blocksMap.removeBlock(block); + removeBlockFromMap(block); // Remove the block from pendingReplications and neededReplications pendingReplications.remove(block); neededReplications.remove(block, UnderReplicatedBlocks.LEVEL); @@ -3559,11 +3558,30 @@ public class BlockManager { } public void removeBlockFromMap(Block block) { + removeFromExcessReplicateMap(block); blocksMap.removeBlock(block); // If block is removed from blocksMap remove it from corruptReplicasMap corruptReplicas.removeFromCorruptReplicasMap(block); } + /** + * If a block is removed from blocksMap, remove it from excessReplicateMap. + */ + private void removeFromExcessReplicateMap(Block block) { + for (DatanodeStorageInfo info : blocksMap.getStorages(block)) { + String uuid = info.getDatanodeDescriptor().getDatanodeUuid(); + LightWeightLinkedSet<Block> excessReplicas = excessReplicateMap.get(uuid); + if (excessReplicas != null) { + if (excessReplicas.remove(block)) { + excessBlocksCount.decrementAndGet(); + if (excessReplicas.isEmpty()) { + excessReplicateMap.remove(uuid); + } + } + } + } + } + public int getCapacity() { return blocksMap.getCapacity(); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/3e793224/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java index 63ab395..1fbb62c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java @@ -265,12 +265,17 @@ public class TestNameNodeMetrics { public void testExcessBlocks() throws Exception { Path file = getTestPath("testExcessBlocks"); createFile(file, 100, (short)2); - long totalBlocks = 1; NameNodeAdapter.setReplication(namesystem, file.toString(), (short)1); updateMetrics(); MetricsRecordBuilder rb = getMetrics(NS_METRICS); - assertGauge("ExcessBlocks", totalBlocks, rb); + assertGauge("ExcessBlocks", 1L, rb); + + // verify ExcessBlocks metric is decremented and + // excessReplicateMap is cleared after deleting a file fs.delete(file, true); + rb = getMetrics(NS_METRICS); + assertGauge("ExcessBlocks", 0L, rb); + assertTrue(bm.excessReplicateMap.isEmpty()); } /** Test to ensure metrics reflects missing blocks */
