HDFS-8794. Improve CorruptReplicasMap#corruptReplicasMap. (yliu)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d6d58606 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d6d58606 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d6d58606 Branch: refs/heads/HDFS-7240 Commit: d6d58606b8adf94b208aed5fc2d054b9dd081db1 Parents: ed01dc7 Author: yliu <y...@apache.org> Authored: Tue Jul 21 09:20:22 2015 +0800 Committer: yliu <y...@apache.org> Committed: Tue Jul 21 09:20:22 2015 +0800 ---------------------------------------------------------------------- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 ++ .../blockmanagement/CorruptReplicasMap.java | 19 ++++++++++++++----- .../blockmanagement/TestCorruptReplicaInfo.java | 12 ++++++------ 3 files changed, 22 insertions(+), 11 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/d6d58606/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index cd32c0e..388b553 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -727,6 +727,8 @@ Release 2.8.0 - UNRELEASED HDFS-7314. When the DFSClient lease cannot be renewed, abort open-for-write files rather than the entire DFSClient. (mingma) + HDFS-8794. Improve CorruptReplicasMap#corruptReplicasMap. (yliu) + OPTIMIZATIONS HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than http://git-wip-us.apache.org/repos/asf/hadoop/blob/d6d58606/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/CorruptReplicasMap.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/CorruptReplicasMap.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/CorruptReplicasMap.java index fc2e234..f83cbaf 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/CorruptReplicasMap.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/CorruptReplicasMap.java @@ -17,12 +17,19 @@ */ package org.apache.hadoop.hdfs.server.blockmanagement; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; +import java.util.TreeMap; + import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.server.namenode.NameNode; import org.apache.hadoop.ipc.Server; -import java.util.*; +import com.google.common.annotations.VisibleForTesting; /** * Stores information about all corrupt blocks in the File System. @@ -46,8 +53,8 @@ public class CorruptReplicasMap{ CORRUPTION_REPORTED // client or datanode reported the corruption } - private final SortedMap<Block, Map<DatanodeDescriptor, Reason>> corruptReplicasMap = - new TreeMap<Block, Map<DatanodeDescriptor, Reason>>(); + private final Map<Block, Map<DatanodeDescriptor, Reason>> corruptReplicasMap = + new HashMap<Block, Map<DatanodeDescriptor, Reason>>(); /** * Mark the block belonging to datanode as corrupt. @@ -181,13 +188,15 @@ public class CorruptReplicasMap{ * @return Up to numExpectedBlocks blocks from startingBlockId if it exists * */ - long[] getCorruptReplicaBlockIds(int numExpectedBlocks, + @VisibleForTesting + long[] getCorruptReplicaBlockIdsForTesting(int numExpectedBlocks, Long startingBlockId) { if (numExpectedBlocks < 0 || numExpectedBlocks > 100) { return null; } - Iterator<Block> blockIt = corruptReplicasMap.keySet().iterator(); + Iterator<Block> blockIt = + new TreeMap<>(corruptReplicasMap).keySet().iterator(); // if the starting block id was specified, iterate over keys until // we find the matching block. If we find a matching block, break http://git-wip-us.apache.org/repos/asf/hadoop/blob/d6d58606/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestCorruptReplicaInfo.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestCorruptReplicaInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestCorruptReplicaInfo.java index 21fb54e..4bdaaac 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestCorruptReplicaInfo.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestCorruptReplicaInfo.java @@ -73,9 +73,9 @@ public class TestCorruptReplicaInfo { // Make sure initial values are returned correctly assertEquals("Number of corrupt blocks must initially be 0", 0, crm.size()); - assertNull("Param n cannot be less than 0", crm.getCorruptReplicaBlockIds(-1, null)); - assertNull("Param n cannot be greater than 100", crm.getCorruptReplicaBlockIds(101, null)); - long[] l = crm.getCorruptReplicaBlockIds(0, null); + assertNull("Param n cannot be less than 0", crm.getCorruptReplicaBlockIdsForTesting(-1, null)); + assertNull("Param n cannot be greater than 100", crm.getCorruptReplicaBlockIdsForTesting(101, null)); + long[] l = crm.getCorruptReplicaBlockIdsForTesting(0, null); assertNotNull("n = 0 must return non-null", l); assertEquals("n = 0 must return an empty list", 0, l.length); @@ -118,14 +118,14 @@ public class TestCorruptReplicaInfo { assertTrue("First five block ids not returned correctly ", Arrays.equals(new long[]{0,1,2,3,4}, - crm.getCorruptReplicaBlockIds(5, null))); + crm.getCorruptReplicaBlockIdsForTesting(5, null))); - LOG.info(crm.getCorruptReplicaBlockIds(10, 7L)); + LOG.info(crm.getCorruptReplicaBlockIdsForTesting(10, 7L)); LOG.info(block_ids.subList(7, 18)); assertTrue("10 blocks after 7 not returned correctly ", Arrays.equals(new long[]{8,9,10,11,12,13,14,15,16,17}, - crm.getCorruptReplicaBlockIds(10, 7L))); + crm.getCorruptReplicaBlockIdsForTesting(10, 7L))); }
