Repository: hadoop Updated Branches: refs/heads/branch-2.6.1 c76f5b318 -> 8ed162bcb
HDFS-7980. Incremental BlockReport will dramatically slow down namenode startup. Contributed by Walter Su (cherry picked from commit 4e1f2eb3955a97a70cf127dc97ae49201a90f5e0) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8ed162bc Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8ed162bc Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8ed162bc Branch: refs/heads/branch-2.6.1 Commit: 8ed162bcbf89c5b05f2092731f60ee0a746cf4ca Parents: c76f5b3 Author: Tsz-Wo Nicholas Sze <[email protected]> Authored: Thu May 7 11:36:35 2015 -0700 Committer: Vinod Kumar Vavilapalli <[email protected]> Committed: Mon Sep 7 11:46:33 2015 -0700 ---------------------------------------------------------------------- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 + .../server/blockmanagement/BlockManager.java | 8 +- .../blockmanagement/TestBlockManager.java | 111 +++++++++++++++++++ 3 files changed, 118 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/8ed162bc/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 759a20f..d2f07c2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -153,6 +153,9 @@ Release 2.6.1 - UNRELEASED HDFS-8270. create() always retried with hardcoded timeout when file already exists with open lease (J.Andreina via vinayakumarb) + HDFS-7980. Incremental BlockReport will dramatically slow down namenode + startup. (Walter Su via szetszwo) + Release 2.6.0 - 2014-11-18 INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/8ed162bc/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java index bd9d2f7..846bc92 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java @@ -1820,7 +1820,7 @@ public class BlockManager { return !node.hasStaleStorages(); } - if (storageInfo.numBlocks() == 0) { + if (storageInfo.getBlockReportCount() == 0) { // The first block report can be processed a lot more efficiently than // ordinary block reports. This shortens restart times. processFirstBlockReport(storageInfo, newReport); @@ -2043,7 +2043,7 @@ public class BlockManager { final BlockListAsLongs report) throws IOException { if (report == null) return; assert (namesystem.hasWriteLock()); - assert (storageInfo.numBlocks() == 0); + assert (storageInfo.getBlockReportCount() == 0); BlockReportIterator itBR = report.getBlockReportIterator(); while(itBR.hasNext()) { @@ -2463,14 +2463,14 @@ public class BlockManager { } // just add it - storageInfo.addBlock(storedBlock); + boolean result = storageInfo.addBlock(storedBlock); // Now check for completion of blocks and safe block count int numCurrentReplica = countLiveNodes(storedBlock); if (storedBlock.getBlockUCState() == BlockUCState.COMMITTED && numCurrentReplica >= minReplication) { completeBlock(storedBlock.getBlockCollection(), storedBlock, false); - } else if (storedBlock.isComplete()) { + } else if (storedBlock.isComplete() && result == true) { // check whether safe replication is reached for the block // only complete blocks are counted towards that. // In the case that the block just became complete above, completeBlock() http://git-wip-us.apache.org/repos/asf/hadoop/blob/8ed162bc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java index 7eec52d..eeec73c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java @@ -47,10 +47,15 @@ import org.apache.hadoop.hdfs.protocol.BlockListAsLongs; import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor.BlockTargetPair; import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils; +import org.apache.hadoop.hdfs.server.datanode.FinalizedReplica; +import org.apache.hadoop.hdfs.server.datanode.ReplicaBeingWritten; +import org.apache.hadoop.hdfs.server.datanode.ReplicaInfo; import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter; import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration; import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage; +import org.apache.hadoop.hdfs.server.protocol.ReceivedDeletedBlockInfo; +import org.apache.hadoop.hdfs.server.protocol.StorageReceivedDeletedBlocks; import org.apache.hadoop.ipc.RemoteException; import org.apache.hadoop.net.NetworkTopology; import org.junit.Assert; @@ -653,6 +658,112 @@ public class TestBlockManager { new BlockListAsLongs(null, null), null, false); assertEquals(1, ds.getBlockReportCount()); } + + /** + * test when NN starts and in same mode, it receives an incremental blockReport + * firstly. Then receives first full block report. + */ + @Test + public void testSafeModeIBRBeforeFirstFullBR() throws Exception { + // pretend to be in safemode + doReturn(true).when(fsn).isInStartupSafeMode(); + + DatanodeDescriptor node = nodes.get(0); + DatanodeStorageInfo ds = node.getStorageInfos()[0]; + node.isAlive = true; + DatanodeRegistration nodeReg = new DatanodeRegistration(node, null, null, ""); + + // register new node + bm.getDatanodeManager().registerDatanode(nodeReg); + bm.getDatanodeManager().addDatanode(node); + assertEquals(node, bm.getDatanodeManager().getDatanode(node)); + assertEquals(0, ds.getBlockReportCount()); + // Build a incremental report + List<ReceivedDeletedBlockInfo> rdbiList = + new ArrayList<ReceivedDeletedBlockInfo>(); + // Build a full report + List<Block> blocks = new ArrayList<Block>(); + + // blk_42 is finalized. + long receivedBlockId = 42; // arbitrary + BlockInfo receivedBlock = addBlockToBM(receivedBlockId); + rdbiList.add(new ReceivedDeletedBlockInfo(new Block(receivedBlock), + ReceivedDeletedBlockInfo.BlockStatus.RECEIVED_BLOCK, null)); + blocks.add(new FinalizedReplica(receivedBlock, null, null)); + + // blk_43 is under construction. + long receivingBlockId = 43; + BlockInfo receivingBlock = addUcBlockToBM(receivingBlockId); + rdbiList.add(new ReceivedDeletedBlockInfo(new Block(receivingBlock), + ReceivedDeletedBlockInfo.BlockStatus.RECEIVING_BLOCK, null)); + blocks.add(new ReplicaBeingWritten(receivingBlock, null, null, null)); + + // blk_44 has 2 records in IBR. It's finalized. So full BR has 1 record. + long receivingReceivedBlockId = 44; + BlockInfo receivingReceivedBlock = addBlockToBM(receivingReceivedBlockId); + rdbiList.add(new ReceivedDeletedBlockInfo(new Block(receivingReceivedBlock), + ReceivedDeletedBlockInfo.BlockStatus.RECEIVING_BLOCK, null)); + rdbiList.add(new ReceivedDeletedBlockInfo(new Block(receivingReceivedBlock), + ReceivedDeletedBlockInfo.BlockStatus.RECEIVED_BLOCK, null)); + blocks.add(new FinalizedReplica(receivingReceivedBlock, null, null)); + + // blk_45 is not in full BR, because it's deleted. + long ReceivedDeletedBlockId = 45; + rdbiList.add(new ReceivedDeletedBlockInfo( + new Block(ReceivedDeletedBlockId), + ReceivedDeletedBlockInfo.BlockStatus.RECEIVED_BLOCK, null)); + rdbiList.add(new ReceivedDeletedBlockInfo( + new Block(ReceivedDeletedBlockId), + ReceivedDeletedBlockInfo.BlockStatus.DELETED_BLOCK, null)); + + // blk_46 exists in DN for a long time, so it's in full BR, but not in IBR. + long existedBlockId = 46; + BlockInfo existedBlock = addBlockToBM(existedBlockId); + blocks.add(new FinalizedReplica(existedBlock, null, null)); + + // process IBR and full BR + StorageReceivedDeletedBlocks srdb = + new StorageReceivedDeletedBlocks(new DatanodeStorage(ds.getStorageID()), + rdbiList.toArray(new ReceivedDeletedBlockInfo[rdbiList.size()])); + bm.processIncrementalBlockReport(node, srdb); + // Make sure it's the first full report + assertEquals(0, ds.getBlockReportCount()); + bm.processReport(node, new DatanodeStorage(ds.getStorageID()), + new BlockListAsLongs(blocks, + (List<ReplicaInfo>) new ArrayList<ReplicaInfo>()), null, false); + assertEquals(1, ds.getBlockReportCount()); + + // verify the storage info is correct + assertTrue(bm.getStoredBlock(new Block(receivedBlockId)).findStorageInfo + (ds) >= 0); + assertTrue(((BlockInfoUnderConstruction) bm. + getStoredBlock(new Block(receivingBlockId))).getNumExpectedLocations() > 0); + assertTrue(bm.getStoredBlock(new Block(receivingReceivedBlockId)) + .findStorageInfo(ds) >= 0); + assertNull(bm.getStoredBlock(new Block(ReceivedDeletedBlockId))); + assertTrue(bm.getStoredBlock(new Block(existedBlock)).findStorageInfo + (ds) >= 0); + } + + private BlockInfo addBlockToBM(long blkId) { + Block block = new Block(blkId); + BlockInfo blockInfo = + new BlockInfo(block, (short) 3); + BlockCollection bc = Mockito.mock(BlockCollection.class); + Mockito.doReturn((short) 3).when(bc).getBlockReplication(); + bm.blocksMap.addBlockCollection(blockInfo, bc); + return blockInfo; + } + + private BlockInfo addUcBlockToBM(long blkId) { + Block block = new Block(blkId); + BlockInfoUnderConstruction blockInfo = + new BlockInfoUnderConstruction(block, (short) 3); + BlockCollection bc = Mockito.mock(BlockCollection.class); + Mockito.doReturn((short) 3).when(bc).getBlockReplication(); + bm.blocksMap.addBlockCollection(blockInfo, bc); + return blockInfo; + } /** * Tests that a namenode doesn't choose a datanode with full disks to
