Repository: hadoop Updated Branches: refs/heads/branch-2 3e8544c5f -> ca6a21f7d
HDFS-7165. Separate block metrics for files with replication count 1. Contributed by Zhe Zhang. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ca6a21f7 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ca6a21f7 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ca6a21f7 Branch: refs/heads/branch-2 Commit: ca6a21f7d1eb57dfd62de03fd9d93069522bb933 Parents: 3e8544c Author: Andrew Wang <[email protected]> Authored: Thu Oct 30 10:21:12 2014 -0700 Committer: Andrew Wang <[email protected]> Committed: Thu Oct 30 10:21:12 2014 -0700 ---------------------------------------------------------------------- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++ .../java/org/apache/hadoop/hdfs/DFSClient.java | 10 ++++++ .../hadoop/hdfs/DistributedFileSystem.java | 10 ++++++ .../hadoop/hdfs/protocol/ClientProtocol.java | 4 ++- .../apache/hadoop/hdfs/protocolPB/PBHelper.java | 7 +++- .../server/blockmanagement/BlockManager.java | 5 +++ .../blockmanagement/HeartbeatManager.java | 2 +- .../blockmanagement/UnderReplicatedBlocks.java | 36 ++++++++++++++++++-- .../hdfs/server/namenode/FSNamesystem.java | 14 ++++++++ .../hdfs/server/namenode/NameNodeMXBean.java | 11 +++++- .../org/apache/hadoop/hdfs/tools/DFSAdmin.java | 2 ++ .../src/main/proto/ClientNamenodeProtocol.proto | 1 + .../hadoop/hdfs/TestMissingBlocksAlert.java | 19 ++++++++++- .../TestUnderReplicatedBlockQueues.java | 19 +++++++++-- .../namenode/metrics/TestNameNodeMetrics.java | 1 + 15 files changed, 135 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/ca6a21f7/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 18d569f..8cd9f69 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -45,6 +45,9 @@ Release 2.7.0 - UNRELEASED HDFS-6824. Additional user documentation for HDFS encryption. (wang) + HDFS-7165. Separate block metrics for files with replication count 1. + (Zhe Zhang via wang) + HDFS-7222. Expose DataNode network errors as a metric. (Charles Lamb via wang) HDFS-7257. Add the time of last HA state transition to NN's /jmx page. http://git-wip-us.apache.org/repos/asf/hadoop/blob/ca6a21f7/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java index 1f66b20..902fc1c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java @@ -2364,6 +2364,16 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory, } /** + * Returns count of blocks with replication factor 1 and have + * lost the only replica. + * @throws IOException + */ + public long getMissingReplOneBlocksCount() throws IOException { + return namenode.getStats()[ClientProtocol. + GET_STATS_MISSING_REPL_ONE_BLOCKS_IDX]; + } + + /** * Returns count of blocks with one of more replica missing. * @throws IOException */ http://git-wip-us.apache.org/repos/asf/hadoop/blob/ca6a21f7/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java index e796a5c..067adce 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java @@ -975,6 +975,16 @@ public class DistributedFileSystem extends FileSystem { } /** + * Returns count of blocks with replication factor 1 and have + * lost the only replica. + * + * @throws IOException + */ + public long getMissingReplOneBlocksCount() throws IOException { + return dfs.getMissingReplOneBlocksCount(); + } + + /** * Returns count of blocks with one of more replica missing. * * @throws IOException http://git-wip-us.apache.org/repos/asf/hadoop/blob/ca6a21f7/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java index d29d2eb..6506bef 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java @@ -652,6 +652,7 @@ public interface ClientProtocol { public int GET_STATS_UNDER_REPLICATED_IDX = 3; public int GET_STATS_CORRUPT_BLOCKS_IDX = 4; public int GET_STATS_MISSING_BLOCKS_IDX = 5; + public int GET_STATS_MISSING_REPL_ONE_BLOCKS_IDX = 6; /** * Get a set of statistics about the filesystem. @@ -663,7 +664,8 @@ public interface ClientProtocol { * <li> [3] contains number of under replicated blocks in the system.</li> * <li> [4] contains number of blocks with a corrupt replica. </li> * <li> [5] contains number of blocks without any good replicas left. </li> - * <li> [6] contains the total used space of the block pool. </li> + * <li> [6] contains number of blocks which have replication factor + * 1 and have lost the only replica. </li> * </ul> * Use public constants like {@link #GET_STATS_CAPACITY_IDX} in place of * actual numbers to index into the array. http://git-wip-us.apache.org/repos/asf/hadoop/blob/ca6a21f7/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java index 64355a9..52f10a2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java @@ -1548,13 +1548,15 @@ public class PBHelper { } public static long[] convert(GetFsStatsResponseProto res) { - long[] result = new long[6]; + long[] result = new long[7]; result[ClientProtocol.GET_STATS_CAPACITY_IDX] = res.getCapacity(); result[ClientProtocol.GET_STATS_USED_IDX] = res.getUsed(); result[ClientProtocol.GET_STATS_REMAINING_IDX] = res.getRemaining(); result[ClientProtocol.GET_STATS_UNDER_REPLICATED_IDX] = res.getUnderReplicated(); result[ClientProtocol.GET_STATS_CORRUPT_BLOCKS_IDX] = res.getCorruptBlocks(); result[ClientProtocol.GET_STATS_MISSING_BLOCKS_IDX] = res.getMissingBlocks(); + result[ClientProtocol.GET_STATS_MISSING_REPL_ONE_BLOCKS_IDX] = + res.getMissingReplOneBlocks(); return result; } @@ -1576,6 +1578,9 @@ public class PBHelper { if (fsStats.length >= ClientProtocol.GET_STATS_MISSING_BLOCKS_IDX + 1) result.setMissingBlocks( fsStats[ClientProtocol.GET_STATS_MISSING_BLOCKS_IDX]); + if (fsStats.length >= ClientProtocol.GET_STATS_MISSING_REPL_ONE_BLOCKS_IDX + 1) + result.setMissingReplOneBlocks( + fsStats[ClientProtocol.GET_STATS_MISSING_REPL_ONE_BLOCKS_IDX]); return result.build(); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/ca6a21f7/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java index 7525522..3f343f3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java @@ -3443,6 +3443,11 @@ public class BlockManager { return this.neededReplications.getCorruptBlockSize(); } + public long getMissingReplOneBlocksCount() { + // not locking + return this.neededReplications.getCorruptReplOneBlockSize(); + } + public BlockInfo addBlockCollection(BlockInfo block, BlockCollection bc) { return blocksMap.addBlockCollection(block, bc); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/ca6a21f7/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/HeartbeatManager.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/HeartbeatManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/HeartbeatManager.java index e53d334..aa4f2f8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/HeartbeatManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/HeartbeatManager.java @@ -179,7 +179,7 @@ class HeartbeatManager implements DatanodeStatistics { -1L, -1L, -1L, - getBlockPoolUsed()}; + -1L}; } @Override http://git-wip-us.apache.org/repos/asf/hadoop/blob/ca6a21f7/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/UnderReplicatedBlocks.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/UnderReplicatedBlocks.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/UnderReplicatedBlocks.java index 8d20016..8da5aee 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/UnderReplicatedBlocks.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/UnderReplicatedBlocks.java @@ -87,7 +87,9 @@ class UnderReplicatedBlocks implements Iterable<Block> { /** Stores the replication index for each priority */ private Map<Integer, Integer> priorityToReplIdx = new HashMap<Integer, Integer>(LEVEL); - + /** The number of corrupt blocks with replication factor 1 */ + private int corruptReplOneBlocks = 0; + /** Create an object. */ UnderReplicatedBlocks() { for (int i = 0; i < LEVEL; i++) { @@ -130,6 +132,11 @@ class UnderReplicatedBlocks implements Iterable<Block> { return priorityQueues.get(QUEUE_WITH_CORRUPT_BLOCKS).size(); } + /** Return the number of corrupt blocks with replication factor 1 */ + synchronized int getCorruptReplOneBlockSize() { + return corruptReplOneBlocks; + } + /** Check if a block is in the neededReplication queue */ synchronized boolean contains(Block block) { for(LightWeightLinkedSet<Block> set : priorityQueues) { @@ -191,6 +198,10 @@ class UnderReplicatedBlocks implements Iterable<Block> { int priLevel = getPriority(block, curReplicas, decomissionedReplicas, expectedReplicas); if(priorityQueues.get(priLevel).add(block)) { + if (priLevel == QUEUE_WITH_CORRUPT_BLOCKS && + expectedReplicas == 1) { + corruptReplOneBlocks++; + } if(NameNode.blockStateChangeLog.isDebugEnabled()) { NameNode.blockStateChangeLog.debug( "BLOCK* NameSystem.UnderReplicationBlock.add:" @@ -213,7 +224,16 @@ class UnderReplicatedBlocks implements Iterable<Block> { int priLevel = getPriority(block, oldReplicas, decommissionedReplicas, oldExpectedReplicas); - return remove(block, priLevel); + boolean removedBlock = remove(block, priLevel); + if (priLevel == QUEUE_WITH_CORRUPT_BLOCKS && + oldExpectedReplicas == 1 && + removedBlock) { + corruptReplOneBlocks--; + assert corruptReplOneBlocks >= 0 : + "Number of corrupt blocks with replication factor 1 " + + "should be non-negative"; + } + return removedBlock; } /** @@ -307,6 +327,18 @@ class UnderReplicatedBlocks implements Iterable<Block> { + " at priority level " + curPri); } } + if (oldPri != curPri || expectedReplicasDelta != 0) { + // corruptReplOneBlocks could possibly change + if (curPri == QUEUE_WITH_CORRUPT_BLOCKS && + curExpectedReplicas == 1) { + // add a new corrupt block with replication factor 1 + corruptReplOneBlocks++; + } else if (oldPri == QUEUE_WITH_CORRUPT_BLOCKS && + curExpectedReplicas - expectedReplicasDelta == 1) { + // remove an existing corrupt block with replication factor 1 + corruptReplOneBlocks--; + } + } } /** http://git-wip-us.apache.org/repos/asf/hadoop/blob/ca6a21f7/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index 6c45ac5..2e39e3b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -5310,6 +5310,13 @@ public class FSNamesystem implements Namesystem, FSClusterStats, // not locking return blockManager.getMissingBlocksCount(); } + + @Metric({"MissingReplOneBlocks", "Number of missing blocks " + + "with replication factor 1"}) + public long getMissingReplOneBlocksCount() { + // not locking + return blockManager.getMissingReplOneBlocksCount(); + } @Metric({"ExpiredHeartbeats", "Number of expired heartbeats"}) public int getExpiredHeartbeats() { @@ -5351,6 +5358,8 @@ public class FSNamesystem implements Namesystem, FSClusterStats, stats[ClientProtocol.GET_STATS_UNDER_REPLICATED_IDX] = getUnderReplicatedBlocks(); stats[ClientProtocol.GET_STATS_CORRUPT_BLOCKS_IDX] = getCorruptReplicaBlocks(); stats[ClientProtocol.GET_STATS_MISSING_BLOCKS_IDX] = getMissingBlocksCount(); + stats[ClientProtocol.GET_STATS_MISSING_REPL_ONE_BLOCKS_IDX] = + getMissingReplOneBlocksCount(); return stats; } @@ -7618,6 +7627,11 @@ public class FSNamesystem implements Namesystem, FSClusterStats, } @Override // NameNodeMXBean + public long getNumberOfMissingBlocksWithReplicationFactorOne() { + return getMissingReplOneBlocksCount(); + } + + @Override // NameNodeMXBean public int getThreads() { return ManagementFactory.getThreadMXBean().getThreadCount(); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/ca6a21f7/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeMXBean.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeMXBean.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeMXBean.java index c9b4d55..3c317fd 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeMXBean.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeMXBean.java @@ -147,11 +147,20 @@ public interface NameNodeMXBean { /** * Gets the total number of missing blocks on the cluster * - * @return the total number of files and blocks on the cluster + * @return the total number of missing blocks on the cluster */ public long getNumberOfMissingBlocks(); /** + * Gets the total number of missing blocks on the cluster with + * replication factor 1 + * + * @return the total number of missing blocks on the cluster with + * replication factor 1 + */ + public long getNumberOfMissingBlocksWithReplicationFactorOne(); + + /** * Gets the number of threads. * * @return the number of threads http://git-wip-us.apache.org/repos/asf/hadoop/blob/ca6a21f7/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java index 04d43a9..fed516b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java @@ -462,6 +462,8 @@ public class DFSAdmin extends FsShell { dfs.getCorruptBlocksCount()); System.out.println("Missing blocks: " + dfs.getMissingBlocksCount()); + System.out.println("Missing blocks (with replication factor 1): " + + dfs.getMissingReplOneBlocksCount()); System.out.println(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/ca6a21f7/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/ClientNamenodeProtocol.proto ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/ClientNamenodeProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/ClientNamenodeProtocol.proto index 232c264..ad75865 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/ClientNamenodeProtocol.proto +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/ClientNamenodeProtocol.proto @@ -283,6 +283,7 @@ message GetFsStatsResponseProto { required uint64 under_replicated = 4; required uint64 corrupt_blocks = 5; required uint64 missing_blocks = 6; + optional uint64 missing_repl_one_blocks = 7; } enum DatanodeReportTypeProto { // type of the datanode report http://git-wip-us.apache.org/repos/asf/hadoop/blob/ca6a21f7/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMissingBlocksAlert.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMissingBlocksAlert.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMissingBlocksAlert.java index 099d547..34b2188 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMissingBlocksAlert.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMissingBlocksAlert.java @@ -77,7 +77,6 @@ public class TestMissingBlocksAlert { Path corruptFile = new Path("/testMissingBlocks/corruptFile"); DFSTestUtil.createFile(dfs, corruptFile, fileLen, (short)3, 0); - // Corrupt the block ExtendedBlock block = DFSTestUtil.getFirstBlock(dfs, corruptFile); assertTrue(TestDatanodeBlockScanner.corruptReplica(block, 0)); @@ -120,6 +119,24 @@ public class TestMissingBlocksAlert { Assert.assertEquals(0, (long)(Long) mbs.getAttribute(mxbeanName, "NumberOfMissingBlocks")); + + Path replOneFile = new Path("/testMissingBlocks/replOneFile"); + DFSTestUtil.createFile(dfs, replOneFile, fileLen, (short)1, 0); + ExtendedBlock replOneBlock = DFSTestUtil.getFirstBlock( + dfs, replOneFile); + assertTrue(TestDatanodeBlockScanner.corruptReplica( + replOneBlock, 0)); + + // read the file so that the corrupt block is reported to NN + in = dfs.open(replOneFile); + try { + in.readFully(new byte[fileLen]); + } catch (ChecksumException ignored) { // checksum error is expected. + } + in.close(); + assertEquals(1, dfs.getMissingReplOneBlocksCount()); + Assert.assertEquals(1, (long)(Long) mbs.getAttribute(mxbeanName, + "NumberOfMissingBlocksWithReplicationFactorOne")); } finally { if (cluster != null) { cluster.shutdown(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/ca6a21f7/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestUnderReplicatedBlockQueues.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestUnderReplicatedBlockQueues.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestUnderReplicatedBlockQueues.java index 20c2541..e87a043 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestUnderReplicatedBlockQueues.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestUnderReplicatedBlockQueues.java @@ -19,10 +19,14 @@ package org.apache.hadoop.hdfs.server.blockmanagement; import org.apache.hadoop.hdfs.protocol.Block; -import org.junit.Assert; import org.junit.Test; -public class TestUnderReplicatedBlockQueues extends Assert { +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.fail; + +public class TestUnderReplicatedBlockQueues { /** * Test that adding blocks with different replication counts puts them @@ -36,6 +40,7 @@ public class TestUnderReplicatedBlockQueues extends Assert { Block block2 = new Block(2); Block block_very_under_replicated = new Block(3); Block block_corrupt = new Block(4); + Block block_corrupt_repl_one = new Block(5); //add a block with a single entry assertAdded(queues, block1, 1, 0, 3); @@ -64,6 +69,16 @@ public class TestUnderReplicatedBlockQueues extends Assert { assertInLevel(queues, block_very_under_replicated, UnderReplicatedBlocks.QUEUE_VERY_UNDER_REPLICATED); + //insert a corrupt block with replication factor 1 + assertAdded(queues, block_corrupt_repl_one, 0, 0, 1); + assertEquals(2, queues.getCorruptBlockSize()); + assertEquals(1, queues.getCorruptReplOneBlockSize()); + queues.update(block_corrupt_repl_one, 0, 0, 3, 0, 2); + assertEquals(0, queues.getCorruptReplOneBlockSize()); + queues.update(block_corrupt, 0, 0, 1, 0, -2); + assertEquals(1, queues.getCorruptReplOneBlockSize()); + queues.update(block_very_under_replicated, 0, 0, 1, -4, -24); + assertEquals(2, queues.getCorruptReplOneBlockSize()); } private void assertAdded(UnderReplicatedBlocks queues, http://git-wip-us.apache.org/repos/asf/hadoop/blob/ca6a21f7/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java index b8d58fa..23d26d6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java @@ -289,6 +289,7 @@ public class TestNameNodeMetrics { MetricsRecordBuilder rb = getMetrics(NS_METRICS); assertGauge("UnderReplicatedBlocks", 1L, rb); assertGauge("MissingBlocks", 1L, rb); + assertGauge("MissingReplOneBlocks", 1L, rb); fs.delete(file, true); waitForDnMetricValue(NS_METRICS, "UnderReplicatedBlocks", 0L); }
