Repository: hadoop Updated Branches: refs/heads/trunk 2e9daa2e2 -> 8c0769dee
HDFS-11960. Successfully closed files can stay under-replicated. Contributed by Kihwal Lee. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8c0769de Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8c0769de Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8c0769de Branch: refs/heads/trunk Commit: 8c0769dee4b455f4de08ccce36334f0be9e79e2c Parents: 2e9daa2 Author: Kihwal Lee <kih...@apache.org> Authored: Tue Jun 20 09:11:23 2017 -0500 Committer: Kihwal Lee <kih...@apache.org> Committed: Tue Jun 20 09:11:23 2017 -0500 ---------------------------------------------------------------------- .../server/blockmanagement/BlockManager.java | 3 +- .../TestPendingReconstruction.java | 44 ++++++++++++++++++++ 2 files changed, 46 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/8c0769de/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java index 2ef80a9..acd4c47 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java @@ -3749,7 +3749,8 @@ public class BlockManager implements BlockStatsMXBean { // Modify the blocks->datanode map and node's map. // BlockInfo storedBlock = getStoredBlock(block); - if (storedBlock != null) { + if (storedBlock != null && + block.getGenerationStamp() == storedBlock.getGenerationStamp()) { pendingReconstruction.decrement(storedBlock, node); } processAndHandleReportedBlock(storageInfo, block, ReplicaState.FINALIZED, http://git-wip-us.apache.org/repos/asf/hadoop/blob/8c0769de/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingReconstruction.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingReconstruction.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingReconstruction.java index aeaaf4e..7679f9d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingReconstruction.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingReconstruction.java @@ -209,6 +209,8 @@ public class TestPendingReconstruction { // Place into blocksmap with GenerationStamp = 1 blockInfo.setGenerationStamp(1); blocksMap.addBlockCollection(blockInfo, bc); + //Save it for later. + BlockInfo storedBlock = blockInfo; assertEquals("Size of pendingReconstructions ", 1, pendingReconstruction.size()); @@ -255,6 +257,48 @@ public class TestPendingReconstruction { // Verify size of neededReconstruction is exactly 1. assertEquals("size of neededReconstruction is 1 ", 1, neededReconstruction.size()); + + // Verify HDFS-11960 + // Stop the replication/redundancy monitor + BlockManagerTestUtil.stopRedundancyThread(blkManager); + pendingReconstruction.clear(); + // Pick a real node + DatanodeDescriptor desc[] = { blkManager.getDatanodeManager(). + getDatanodes().iterator().next() }; + + // Add a stored block to the pendingReconstruction. + pendingReconstruction.increment(storedBlock, desc); + assertEquals("Size of pendingReconstructions ", 1, + pendingReconstruction.size()); + + // A received IBR processing calls addBlock(). If the gen stamp in the + // report is not the same, it should stay in pending. + fsn.writeLock(); + try { + // Use a wrong gen stamp. + blkManager.addBlock(desc[0].getStorageInfos()[0], + new Block(1, 1, 0), null); + } finally { + fsn.writeUnlock(); + } + + // The block should still be pending + assertEquals("Size of pendingReconstructions ", 1, + pendingReconstruction.size()); + + // A block report with the correct gen stamp should remove the record + // from the pending queue. + fsn.writeLock(); + try { + blkManager.addBlock(desc[0].getStorageInfos()[0], + new Block(1, 1, 1), null); + } finally { + fsn.writeUnlock(); + } + + // The pending queue should be empty. + assertEquals("Size of pendingReconstructions ", 0, + pendingReconstruction.size()); } finally { if (cluster != null) { cluster.shutdown(); --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
