HDFS-13418. NetworkTopology should be configurable when enable DFSNetworkTopology. Contributed by Tao Jie.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0725953e Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0725953e Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0725953e Branch: refs/heads/HDFS-7240 Commit: 0725953efec89b35b7586b846abb01f7c5963b37 Parents: 1a407bc Author: Yiqun Lin <yq...@apache.org> Authored: Fri Apr 13 17:55:45 2018 +0800 Committer: Yiqun Lin <yq...@apache.org> Committed: Fri Apr 13 17:55:45 2018 +0800 ---------------------------------------------------------------------- .../org/apache/hadoop/hdfs/DFSConfigKeys.java | 7 +++ .../hadoop/hdfs/net/DFSNetworkTopology.java | 10 +++- .../src/main/resources/hdfs-default.xml | 14 ++++++ .../blockmanagement/TestDatanodeManager.java | 52 ++++++++++++++++++++ 4 files changed, 81 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/0725953e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java index b4b9d97..b4dab4d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java @@ -21,6 +21,7 @@ package org.apache.hadoop.hdfs; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.fs.CommonConfigurationKeys; import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys; +import org.apache.hadoop.hdfs.net.DFSNetworkTopology; import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicyDefault; import org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicyRackFaultTolerant; @@ -1177,6 +1178,12 @@ public class DFSConfigKeys extends CommonConfigurationKeys { "dfs.use.dfs.network.topology"; public static final boolean DFS_USE_DFS_NETWORK_TOPOLOGY_DEFAULT = true; + public static final String DFS_NET_TOPOLOGY_IMPL_KEY = + "dfs.net.topology.impl"; + + public static final Class<DFSNetworkTopology> DFS_NET_TOPOLOGY_IMPL_DEFAULT = + DFSNetworkTopology.class; + // dfs.client.retry confs are moved to HdfsClientConfigKeys.Retry @Deprecated public static final String DFS_CLIENT_RETRY_POLICY_ENABLED_KEY http://git-wip-us.apache.org/repos/asf/hadoop/blob/0725953e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/net/DFSNetworkTopology.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/net/DFSNetworkTopology.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/net/DFSNetworkTopology.java index e74cdec..7889ef4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/net/DFSNetworkTopology.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/net/DFSNetworkTopology.java @@ -22,11 +22,13 @@ import com.google.common.base.Preconditions; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.StorageType; +import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor; import org.apache.hadoop.net.NetworkTopology; import org.apache.hadoop.net.Node; import org.apache.hadoop.net.NodeBase; +import org.apache.hadoop.util.ReflectionUtils; import java.util.ArrayList; import java.util.Collection; @@ -44,8 +46,12 @@ public class DFSNetworkTopology extends NetworkTopology { private static final Random RANDOM = new Random(); public static DFSNetworkTopology getInstance(Configuration conf) { - DFSNetworkTopology nt = new DFSNetworkTopology(); - return (DFSNetworkTopology)nt.init(DFSTopologyNodeImpl.FACTORY); + + DFSNetworkTopology nt = ReflectionUtils.newInstance(conf.getClass( + DFSConfigKeys.DFS_NET_TOPOLOGY_IMPL_KEY, + DFSConfigKeys.DFS_NET_TOPOLOGY_IMPL_DEFAULT, + DFSNetworkTopology.class), conf); + return (DFSNetworkTopology) nt.init(DFSTopologyNodeImpl.FACTORY); } /** http://git-wip-us.apache.org/repos/asf/hadoop/blob/0725953e/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml index f8cce60..4ed4690 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml @@ -4859,6 +4859,20 @@ <value>true</value> <description> Enables DFSNetworkTopology to choose nodes for placing replicas. + When enabled, NetworkTopology will be instantiated as class defined in + property dfs.net.topology.impl, otherwise NetworkTopology will be + instantiated as class defined in property net.topology.impl. + </description> + </property> + + <property> + <name>dfs.net.topology.impl</name> + <value>org.apache.hadoop.hdfs.net.DFSNetworkTopology</value> + <description> + The implementation class of NetworkTopology used in HDFS. By default, + the class org.apache.hadoop.hdfs.net.DFSNetworkTopology is specified and + used in block placement. + This property only works when dfs.use.dfs.network.topology is true. </description> </property> http://git-wip-us.apache.org/repos/asf/hadoop/blob/0725953e/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestDatanodeManager.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestDatanodeManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestDatanodeManager.java index 96841cf..f708740 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestDatanodeManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestDatanodeManager.java @@ -41,6 +41,8 @@ import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.fs.StorageType; import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.hdfs.HdfsConfiguration; +import org.apache.hadoop.hdfs.net.DFSNetworkTopology; import org.apache.hadoop.hdfs.protocol.DatanodeID; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.protocol.ExtendedBlock; @@ -61,11 +63,13 @@ import org.apache.hadoop.hdfs.server.protocol.SlowDiskReports; import org.apache.hadoop.hdfs.server.protocol.SlowPeerReports; import org.apache.hadoop.hdfs.server.protocol.StorageReport; import org.apache.hadoop.net.DNSToSwitchMapping; +import org.apache.hadoop.net.NetworkTopology; import org.apache.hadoop.util.Shell; import org.junit.Assert; import org.junit.Test; import org.mockito.Mockito; import org.mockito.internal.util.reflection.Whitebox; + import static org.hamcrest.core.Is.is; import static org.junit.Assert.*; @@ -622,4 +626,52 @@ public class TestDatanodeManager { // Approximately load tasks if the ratio between queue length is large. verifyPendingRecoveryTasks(400, 1, 20, 20, 1); } + + @Test + public void testNetworkTopologyInstantiation() throws Exception { + // case 1, dfs.use.dfs.network.topology=true, use the default + // DFSNetworkTopology impl. + Configuration conf1 = new HdfsConfiguration(); + FSNamesystem fsn = Mockito.mock(FSNamesystem.class); + DatanodeManager dm1 = mockDatanodeManager(fsn, conf1); + assertEquals(DFSNetworkTopology.class, dm1.getNetworkTopology().getClass()); + + // case 2, dfs.use.dfs.network.topology=false, use the default + // NetworkTopology impl. + Configuration conf2 = new HdfsConfiguration(); + conf2.setBoolean(DFSConfigKeys.DFS_USE_DFS_NETWORK_TOPOLOGY_KEY, false); + DatanodeManager dm2 = mockDatanodeManager(fsn, conf2); + assertEquals(NetworkTopology.class, dm2.getNetworkTopology() + .getClass()); + + // case 3, dfs.use.dfs.network.topology=false, and specify the + // net.topology.impl property. + Configuration conf3 = new HdfsConfiguration(); + conf3.setClass(CommonConfigurationKeysPublic.NET_TOPOLOGY_IMPL_KEY, + MockDfsNetworkTopology.class, NetworkTopology.class); + conf3.setBoolean(DFSConfigKeys.DFS_USE_DFS_NETWORK_TOPOLOGY_KEY, false); + DatanodeManager dm3 = mockDatanodeManager(fsn, conf3); + assertEquals(MockDfsNetworkTopology.class, dm3.getNetworkTopology() + .getClass()); + + // case 4, dfs.use.dfs.network.topology=true, and specify the + // dfs.net.topology.impl property. + Configuration conf4 = new HdfsConfiguration(); + conf4.setClass(DFSConfigKeys.DFS_NET_TOPOLOGY_IMPL_KEY, + MockDfsNetworkTopology.class, NetworkTopology.class); + conf4.setBoolean(DFSConfigKeys.DFS_USE_DFS_NETWORK_TOPOLOGY_KEY, true); + DatanodeManager dm4 = mockDatanodeManager(fsn, conf4); + assertEquals(MockDfsNetworkTopology.class, dm4.getNetworkTopology() + .getClass()); + } + + /** + * A NetworkTopology implementation for test. + * + */ + public static class MockDfsNetworkTopology extends DFSNetworkTopology { + public MockDfsNetworkTopology(){ + super(); + } + } } --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
