Repository: hadoop Updated Branches: refs/heads/trunk 392965370 -> 27978bcb6
HDFS-13833. Improve BlockPlacementPolicyDefault's consider load logic. Contributed by Shweta. Signed-off-by: Xiao Chen <x...@apache.org> Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/27978bcb Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/27978bcb Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/27978bcb Branch: refs/heads/trunk Commit: 27978bcb66a9130cbf26d37ec454c0b7fcdc2530 Parents: 3929653 Author: Shweta <shw...@cloudera.com> Authored: Tue Sep 18 20:22:25 2018 -0700 Committer: Xiao Chen <x...@apache.org> Committed: Tue Sep 18 20:23:50 2018 -0700 ---------------------------------------------------------------------- .../BlockPlacementPolicyDefault.java | 29 ++++++++++++++------ .../blockmanagement/TestReplicationPolicy.java | 28 +++++++++++++++++++ 2 files changed, 49 insertions(+), 8 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/27978bcb/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java index d00f961..d396845 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java @@ -913,6 +913,24 @@ public class BlockPlacementPolicyDefault extends BlockPlacementPolicy { } /** + * Determine if a datanode should be chosen based on current workload. + * + * @param node The target datanode + * @return Return true if the datanode should be excluded, otherwise false + */ + boolean excludeNodeByLoad(DatanodeDescriptor node){ + final double maxLoad = considerLoadFactor * + stats.getInServiceXceiverAverage(); + final int nodeLoad = node.getXceiverCount(); + if ((nodeLoad > maxLoad) && (maxLoad > 0)) { + logNodeIsNotChosen(node, NodeNotChosenReason.NODE_TOO_BUSY, + "(load: " + nodeLoad + " > " + maxLoad + ")"); + return true; + } + return false; + } + + /** * Determine if a datanode is good for placing block. * * @param node The target datanode @@ -923,7 +941,7 @@ public class BlockPlacementPolicyDefault extends BlockPlacementPolicy { * @param results A list containing currently chosen nodes. Used to check if * too many nodes has been chosen in the target rack. * @param avoidStaleNodes Whether or not to avoid choosing stale nodes - * @return Reture true if the datanode is good candidate, otherwise false + * @return Return true if the datanode is good candidate, otherwise false */ boolean isGoodDatanode(DatanodeDescriptor node, int maxTargetPerRack, boolean considerLoad, @@ -943,13 +961,8 @@ public class BlockPlacementPolicyDefault extends BlockPlacementPolicy { } // check the communication traffic of the target machine - if (considerLoad) { - final double maxLoad = considerLoadFactor * - stats.getInServiceXceiverAverage(); - final int nodeLoad = node.getXceiverCount(); - if (nodeLoad > maxLoad) { - logNodeIsNotChosen(node, NodeNotChosenReason.NODE_TOO_BUSY, - "(load: " + nodeLoad + " > " + maxLoad + ")"); + if(considerLoad){ + if(excludeNodeByLoad(node)){ return false; } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/27978bcb/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java index 27dcbf1..f08fa13 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java @@ -65,6 +65,7 @@ import org.apache.hadoop.hdfs.server.namenode.Namesystem; import org.apache.hadoop.hdfs.server.namenode.TestINodeFile; import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage; import org.apache.hadoop.net.Node; +import org.apache.hadoop.util.ReflectionUtils; import org.apache.log4j.Level; import org.apache.log4j.Logger; import org.apache.log4j.spi.LoggingEvent; @@ -1559,4 +1560,31 @@ public class TestReplicationPolicy extends BaseReplicationPolicyTest { } assertTrue(found); } + + @Test + public void testMaxLoad() { + FSClusterStats statistics = mock(FSClusterStats.class); + DatanodeDescriptor node = mock(DatanodeDescriptor.class); + + when(statistics.getInServiceXceiverAverage()).thenReturn(0.0); + when(node.getXceiverCount()).thenReturn(1); + + final Configuration conf = new Configuration(); + final Class<? extends BlockPlacementPolicy> replicatorClass = conf + .getClass(DFSConfigKeys.DFS_BLOCK_REPLICATOR_CLASSNAME_KEY, + DFSConfigKeys.DFS_BLOCK_REPLICATOR_CLASSNAME_DEFAULT, + BlockPlacementPolicy.class); + BlockPlacementPolicy bpp = ReflectionUtils. + newInstance(replicatorClass, conf); + assertTrue(bpp instanceof BlockPlacementPolicyDefault); + + BlockPlacementPolicyDefault bppd = (BlockPlacementPolicyDefault) bpp; + bppd.initialize(conf, statistics, null, null); + assertFalse(bppd.excludeNodeByLoad(node)); + + when(statistics.getInServiceXceiverAverage()).thenReturn(1.0); + when(node.getXceiverCount()).thenReturn(10); + assertTrue(bppd.excludeNodeByLoad(node)); + + } } --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org