Repository: hadoop Updated Branches: refs/heads/branch-2.7 336be63da -> 13b256ed2
HDFS-9083. Replication violates block placement policy. Contributed by Rushabh Shah. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/13b256ed Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/13b256ed Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/13b256ed Branch: refs/heads/branch-2.7 Commit: 13b256ed2219078653820b544313637ce80a7120 Parents: 336be63 Author: Kihwal Lee <kih...@apache.org> Authored: Wed Oct 28 14:49:41 2015 -0500 Committer: Kihwal Lee <kih...@apache.org> Committed: Wed Oct 28 14:49:41 2015 -0500 ---------------------------------------------------------------------- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 ++ .../server/blockmanagement/BlockManager.java | 10 -------- .../blockmanagement/TestBlockManager.java | 24 ++++++++++++++++++++ 3 files changed, 26 insertions(+), 10 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/13b256ed/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index e63532d..bd6e9f0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -62,6 +62,8 @@ Release 2.7.2 - UNRELEASED HDFS-9043. Doc updation for commands in HDFS Federation (J.Andreina via vinayakumab) + HDFS-9083. Replication violates block placement policy (Rushabh Shah) + HDFS-9106. Transfer failure during pipeline recovery causes permanent write failures (kihwal) http://git-wip-us.apache.org/repos/asf/hadoop/blob/13b256ed/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java index d770346..c360d4c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java @@ -224,9 +224,6 @@ public class BlockManager { final float blocksInvalidateWorkPct; final int blocksReplWorkMultiplier; - - /** variable to enable check for enough racks */ - final boolean shouldCheckForEnoughRacks; // whether or not to issue block encryption keys. final boolean encryptDataTransfer; @@ -325,9 +322,6 @@ public class BlockManager { conf.getInt( DFSConfigKeys.DFS_NAMENODE_REPLICATION_STREAMS_HARD_LIMIT_KEY, DFSConfigKeys.DFS_NAMENODE_REPLICATION_STREAMS_HARD_LIMIT_DEFAULT); - this.shouldCheckForEnoughRacks = - conf.get(DFSConfigKeys.NET_TOPOLOGY_SCRIPT_FILE_NAME_KEY) == null - ? false : true; this.blocksInvalidateWorkPct = DFSUtil.getInvalidateWorkPctPerIteration(conf); this.blocksReplWorkMultiplier = DFSUtil.getReplWorkMultiplier(conf); @@ -351,7 +345,6 @@ public class BlockManager { LOG.info("maxReplication = " + maxReplication); LOG.info("minReplication = " + minReplication); LOG.info("maxReplicationStreams = " + maxReplicationStreams); - LOG.info("shouldCheckForEnoughRacks = " + shouldCheckForEnoughRacks); LOG.info("replicationRecheckInterval = " + replicationRecheckInterval); LOG.info("encryptDataTransfer = " + encryptDataTransfer); LOG.info("maxNumBlocksToLog = " + maxNumBlocksToLog); @@ -3490,9 +3483,6 @@ public class BlockManager { } boolean blockHasEnoughRacks(Block b) { - if (!this.shouldCheckForEnoughRacks) { - return true; - } boolean enoughRacks = false;; Collection<DatanodeDescriptor> corruptNodes = corruptReplicas.getNodes(b); http://git-wip-us.apache.org/repos/asf/hadoop/blob/13b256ed/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java index fba840e..e026a53 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java @@ -816,4 +816,28 @@ public class TestBlockManager { Assert.assertFalse(BlockManager.useDelHint(true, delHint, null, moreThan1Racks, excessTypes)); } + + /** + * {@link BlockManager#blockHasEnoughRacks(BlockInfo)} should return false + * if all the replicas are on the same rack and shouldn't be dependent on + * CommonConfigurationKeysPublic.NET_TOPOLOGY_SCRIPT_FILE_NAME_KEY + * @throws Exception + */ + @Test + public void testAllReplicasOnSameRack() throws Exception { + Configuration conf = new HdfsConfiguration(); + conf.unset(DFSConfigKeys.NET_TOPOLOGY_SCRIPT_FILE_NAME_KEY); + fsn = Mockito.mock(FSNamesystem.class); + Mockito.doReturn(true).when(fsn).hasWriteLock(); + Mockito.doReturn(true).when(fsn).hasReadLock(); + bm = new BlockManager(fsn, conf); + // Add nodes on two racks + addNodes(nodes); + // Added a new block in blocksMap and all the replicas are on the same rack + BlockInfoContiguous blockInfo = addBlockOnNodes(1, rackA); + // Since the network toppolgy is multi-rack, the blockHasEnoughRacks + // should return false. + assertFalse("Replicas for block is not stored on enough racks", + bm.blockHasEnoughRacks(blockInfo)); + } }
