HDFS-11998. Enable DFSNetworkTopology as default. Contributed by Chen Liang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9ae9467f Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9ae9467f Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9ae9467f Branch: refs/heads/HADOOP-13345 Commit: 9ae9467f920e95ca989d7d51775b39e1b9fee300 Parents: c22cf00 Author: Arpit Agarwal <[email protected]> Authored: Wed Jun 21 22:01:37 2017 -0700 Committer: Arpit Agarwal <[email protected]> Committed: Wed Jun 21 22:01:41 2017 -0700 ---------------------------------------------------------------------- .../org/apache/hadoop/hdfs/DFSConfigKeys.java | 2 +- .../AvailableSpaceBlockPlacementPolicy.java | 21 ++++++++++++++++++++ .../src/main/resources/hdfs-default.xml | 2 +- .../TestReplicationPolicyWithNodeGroup.java | 4 ++++ 4 files changed, 27 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/9ae9467f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java index e7aba4e..1f60f32 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java @@ -1085,7 +1085,7 @@ public class DFSConfigKeys extends CommonConfigurationKeys { public static final String DFS_USE_DFS_NETWORK_TOPOLOGY_KEY = "dfs.use.dfs.network.topology"; - public static final boolean DFS_USE_DFS_NETWORK_TOPOLOGY_DEFAULT = false; + public static final boolean DFS_USE_DFS_NETWORK_TOPOLOGY_DEFAULT = true; // dfs.client.retry confs are moved to HdfsClientConfigKeys.Retry @Deprecated http://git-wip-us.apache.org/repos/asf/hadoop/blob/9ae9467f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/AvailableSpaceBlockPlacementPolicy.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/AvailableSpaceBlockPlacementPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/AvailableSpaceBlockPlacementPolicy.java index 706768c..8435b46 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/AvailableSpaceBlockPlacementPolicy.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/AvailableSpaceBlockPlacementPolicy.java @@ -24,10 +24,13 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_AVAILABLE_SPACE_ import java.util.Collection; import java.util.Random; +import com.google.common.base.Preconditions; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.StorageType; import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.hdfs.net.DFSNetworkTopology; import org.apache.hadoop.net.NetworkTopology; import org.apache.hadoop.net.Node; @@ -71,11 +74,29 @@ public class AvailableSpaceBlockPlacementPolicy extends @Override protected DatanodeDescriptor chooseDataNode(final String scope, + final Collection<Node> excludedNode, StorageType type) { + // only the code that uses DFSNetworkTopology should trigger this code path. + Preconditions.checkArgument(clusterMap instanceof DFSNetworkTopology); + DFSNetworkTopology dfsClusterMap = (DFSNetworkTopology)clusterMap; + DatanodeDescriptor a = (DatanodeDescriptor) dfsClusterMap + .chooseRandomWithStorageType(scope, excludedNode, type); + DatanodeDescriptor b = (DatanodeDescriptor) dfsClusterMap + .chooseRandomWithStorageType(scope, excludedNode, type); + return select(a, b); + } + + @Override + protected DatanodeDescriptor chooseDataNode(final String scope, final Collection<Node> excludedNode) { DatanodeDescriptor a = (DatanodeDescriptor) clusterMap.chooseRandom(scope, excludedNode); DatanodeDescriptor b = (DatanodeDescriptor) clusterMap.chooseRandom(scope, excludedNode); + return select(a, b); + } + + private DatanodeDescriptor select( + DatanodeDescriptor a, DatanodeDescriptor b) { if (a != null && b != null){ int ret = compareDataNode(a, b); if (ret == 0) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/9ae9467f/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml index 0f72041..be345af 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml @@ -4534,7 +4534,7 @@ <property> <name>dfs.use.dfs.network.topology</name> - <value>false</value> + <value>true</value> <description> Enables DFSNetworkTopology to choose nodes for placing replicas. </description> http://git-wip-us.apache.org/repos/asf/hadoop/blob/9ae9467f/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyWithNodeGroup.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyWithNodeGroup.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyWithNodeGroup.java index ebd4b81..6572f78 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyWithNodeGroup.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyWithNodeGroup.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.hdfs.server.blockmanagement; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_USE_DFS_NETWORK_TOPOLOGY_KEY; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -51,6 +52,9 @@ public class TestReplicationPolicyWithNodeGroup extends BaseReplicationPolicyTes @Override DatanodeDescriptor[] getDatanodeDescriptors(Configuration conf) { + // default is true, in this case this test will against DFSNetworkTopology + // but it run on NetworkTopologyWithNodeGroup, so set to false. + conf.setBoolean(DFS_USE_DFS_NETWORK_TOPOLOGY_KEY, false); conf.set(CommonConfigurationKeysPublic.NET_TOPOLOGY_IMPL_KEY, NetworkTopologyWithNodeGroup.class.getName()); final String[] racks = { --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
