HDFS-12249. dfsadmin -metaSave to output maintenance mode blocks. Contributed by Wellington Chevreuil.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0f2a6912 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0f2a6912 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0f2a6912 Branch: refs/heads/HDFS-9806 Commit: 0f2a69127aa40b7dccdf74ff63e2205a10ae8998 Parents: 9e77dc2 Author: Wei-Chiu Chuang <[email protected]> Authored: Mon Oct 23 15:23:52 2017 -0700 Committer: Wei-Chiu Chuang <[email protected]> Committed: Mon Oct 23 15:24:34 2017 -0700 ---------------------------------------------------------------------- .../server/blockmanagement/BlockManager.java | 12 +- .../blockmanagement/TestBlockManager.java | 111 +++++++++++++++++++ 2 files changed, 119 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/0f2a6912/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java index 8b0f094..6cd67f6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java @@ -833,10 +833,12 @@ public class BlockManager implements BlockStatsMXBean { // l: == live:, d: == decommissioned c: == corrupt e: == excess out.print(block + ((usableReplicas > 0)? "" : " MISSING") + " (replicas:" + - " l: " + numReplicas.liveReplicas() + - " d: " + numReplicas.decommissionedAndDecommissioning() + - " c: " + numReplicas.corruptReplicas() + - " e: " + numReplicas.excessReplicas() + ") "); + " live: " + numReplicas.liveReplicas() + + " decommissioning and decommissioned: " + + numReplicas.decommissionedAndDecommissioning() + + " corrupt: " + numReplicas.corruptReplicas() + + " in excess: " + numReplicas.excessReplicas() + + " maintenance mode: " + numReplicas.maintenanceReplicas() + ") "); Collection<DatanodeDescriptor> corruptNodes = corruptReplicas.getNodes(block); @@ -849,6 +851,8 @@ public class BlockManager implements BlockStatsMXBean { } else if (node.isDecommissioned() || node.isDecommissionInProgress()) { state = "(decommissioned)"; + } else if (node.isMaintenance() || node.isInMaintenance()){ + state = "(maintenance)"; } if (storage.areBlockContentsStale()) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/0f2a6912/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java index 10289ed..4d01d80 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java @@ -1511,4 +1511,115 @@ public class TestBlockManager { return blockInfo; } + private BlockInfo makeBlockReplicasMaintenance(long blockId, + List<DatanodeDescriptor> nodesList) throws IOException { + long inodeId = ++mockINodeId; + final INodeFile bc = TestINodeFile.createINodeFile(inodeId); + + BlockInfo blockInfo = blockOnNodes(blockId, nodesList); + blockInfo.setReplication((short) 3); + blockInfo.setBlockCollectionId(inodeId); + + Mockito.doReturn(bc).when(fsn).getBlockCollection(inodeId); + bm.blocksMap.addBlockCollection(blockInfo, bc); + nodesList.get(0).setInMaintenance(); + BlockCollection mockedBc = Mockito.mock(BlockCollection.class); + Mockito.when(mockedBc.getBlocks()).thenReturn(new BlockInfo[]{blockInfo}); + bm.checkRedundancy(mockedBc); + return blockInfo; + } + + @Test + public void testMetaSaveInMaintenanceReplicas() throws Exception { + List<DatanodeStorageInfo> origStorages = getStorages(0, 1); + List<DatanodeDescriptor> origNodes = getNodes(origStorages); + BlockInfo block = makeBlockReplicasMaintenance(0, origNodes); + File file = new File("test.log"); + PrintWriter out = new PrintWriter(file); + bm.metaSave(out); + out.flush(); + FileInputStream fstream = new FileInputStream(file); + DataInputStream in = new DataInputStream(fstream); + BufferedReader reader = new BufferedReader(new InputStreamReader(in)); + StringBuffer buffer = new StringBuffer(); + String line; + try { + while ((line = reader.readLine()) != null) { + buffer.append(line); + System.out.println(line); + } + String output = buffer.toString(); + assertTrue("Metasave output should not have reported " + + "missing blocks.", + output.contains("Metasave: Blocks currently missing: 0")); + assertTrue("There should be 1 block waiting for reconstruction", + output.contains("Metasave: Blocks waiting for reconstruction: 1")); + String blockNameGS = block.getBlockName() + "_" + + block.getGenerationStamp(); + assertTrue("Block " + blockNameGS + + " should be list as maintenance.", + output.contains(blockNameGS + " (replicas: live: 1 decommissioning " + + "and decommissioned: 0 corrupt: 0 in excess: " + + "0 maintenance mode: 1)")); + } finally { + reader.close(); + file.delete(); + } + } + + private BlockInfo makeBlockReplicasDecommission(long blockId, + List<DatanodeDescriptor> nodesList) throws IOException { + long inodeId = ++mockINodeId; + final INodeFile bc = TestINodeFile.createINodeFile(inodeId); + + BlockInfo blockInfo = blockOnNodes(blockId, nodesList); + blockInfo.setReplication((short) 3); + blockInfo.setBlockCollectionId(inodeId); + + Mockito.doReturn(bc).when(fsn).getBlockCollection(inodeId); + bm.blocksMap.addBlockCollection(blockInfo, bc); + nodesList.get(0).startDecommission(); + BlockCollection mockedBc = Mockito.mock(BlockCollection.class); + Mockito.when(mockedBc.getBlocks()).thenReturn(new BlockInfo[]{blockInfo}); + bm.checkRedundancy(mockedBc); + return blockInfo; + } + + @Test + public void testMetaSaveDecommissioningReplicas() throws Exception { + List<DatanodeStorageInfo> origStorages = getStorages(0, 1); + List<DatanodeDescriptor> origNodes = getNodes(origStorages); + BlockInfo block = makeBlockReplicasDecommission(0, origNodes); + File file = new File("test.log"); + PrintWriter out = new PrintWriter(file); + bm.metaSave(out); + out.flush(); + FileInputStream fstream = new FileInputStream(file); + DataInputStream in = new DataInputStream(fstream); + BufferedReader reader = new BufferedReader(new InputStreamReader(in)); + StringBuffer buffer = new StringBuffer(); + String line; + try { + while ((line = reader.readLine()) != null) { + buffer.append(line); + } + String output = buffer.toString(); + assertTrue("Metasave output should not have reported " + + "missing blocks.", + output.contains("Metasave: Blocks currently missing: 0")); + assertTrue("There should be 1 block waiting for reconstruction", + output.contains("Metasave: Blocks waiting for reconstruction: 1")); + String blockNameGS = block.getBlockName() + "_" + + block.getGenerationStamp(); + assertTrue("Block " + blockNameGS + + " should be list as maintenance.", + output.contains(blockNameGS + " (replicas: live: 1 decommissioning " + + "and decommissioned: 1 corrupt: 0 in excess: " + + "0 maintenance mode: 0)")); + } finally { + reader.close(); + file.delete(); + } + } + } --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
