Author: szetszwo
Date: Sat May 3 11:02:44 2014
New Revision: 1592179
URL: http://svn.apache.org/r1592179
Log:
HDFS-5168. Add cross node dependency support to BlockPlacementPolicy.
Contributed by Nikola Vujic
Added:
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/DNSToSwitchMappingWithDependency.java
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/ScriptBasedMappingWithDependency.java
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestScriptBasedMappingWithDependency.java
Modified:
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/ScriptBasedMapping.java
Modified:
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java?rev=1592179&r1=1592178&r2=1592179&view=diff
==============================================================================
---
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
(original)
+++
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
Sat May 3 11:02:44 2014
@@ -78,6 +78,8 @@ public class CommonConfigurationKeysPubl
/** See <a href="{@docRoot}/../core-default.html">core-default.xml</a> */
public static final String NET_TOPOLOGY_TABLE_MAPPING_FILE_KEY =
"net.topology.table.file.name";
+ public static final String NET_DEPENDENCY_SCRIPT_FILE_NAME_KEY =
+ "net.topology.dependency.script.file.name";
/** See <a href="{@docRoot}/../core-default.html">core-default.xml</a> */
public static final String FS_TRASH_CHECKPOINT_INTERVAL_KEY =
Added:
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/DNSToSwitchMappingWithDependency.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/DNSToSwitchMappingWithDependency.java?rev=1592179&view=auto
==============================================================================
---
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/DNSToSwitchMappingWithDependency.java
(added)
+++
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/DNSToSwitchMappingWithDependency.java
Sat May 3 11:02:44 2014
@@ -0,0 +1,56 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.net;
+
+import java.util.List;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * An interface that must be implemented to allow pluggable
+ * DNS-name/IP-address to RackID resolvers.
+ *
+ */
[email protected]
[email protected]
+
+public interface DNSToSwitchMappingWithDependency extends DNSToSwitchMapping {
+ /**
+ * Get a list of dependent DNS-names for a given DNS-name/IP-address.
+ * Dependent DNS-names fall into the same fault domain which must be
+ * taken into account when placing replicas. This is intended to be used for
+ * cross node group dependencies when node groups are not sufficient to
+ * distinguish data nodes by fault domains. In practice, this is needed when
+ * a compute server runs VMs which use shared storage (as opposite to
+ * directly attached storage). In this case data nodes fall in two different
+ * fault domains. One fault domain is defined by a compute server and
+ * the other is defined by storage. With node groups we can group data nodes
+ * either by server fault domain or by storage fault domain. However one of
+ * the fault domains cannot be handled and there we need to define cross node
+ * group dependencies. These dependencies are applied in block placement
+ * polices which ensure that no two replicas will be on two dependent nodes.
+ * @param name - host name or IP address of a data node. Input host name
+ * parameter must take a value of dfs.datanode.hostname config value if this
+ * config property is set. Otherwise FQDN of the data node is used.
+ * @return list of dependent host names. If dfs.datanode.hostname config
+ * property is set, then its value must be returned.
+ * Otherwise, FQDN is returned.
+ */
+ public List<String> getDependency(String name);
+}
Modified:
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/ScriptBasedMapping.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/ScriptBasedMapping.java?rev=1592179&r1=1592178&r2=1592179&view=diff
==============================================================================
---
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/ScriptBasedMapping.java
(original)
+++
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/ScriptBasedMapping.java
Sat May 3 11:02:44 2014
@@ -45,7 +45,7 @@ import org.apache.hadoop.fs.CommonConfig
*/
@InterfaceAudience.Public
@InterfaceStability.Evolving
-public final class ScriptBasedMapping extends CachedDNSToSwitchMapping {
+public class ScriptBasedMapping extends CachedDNSToSwitchMapping {
/**
* Minimum number of arguments: {@value}
@@ -63,6 +63,7 @@ public final class ScriptBasedMapping ex
*/
static final String SCRIPT_FILENAME_KEY =
CommonConfigurationKeys.NET_TOPOLOGY_SCRIPT_FILE_NAME_KEY
;
+
/**
* key to the argument count that the script supports
* {@value}
@@ -84,7 +85,15 @@ public final class ScriptBasedMapping ex
*
*/
public ScriptBasedMapping() {
- super(new RawScriptBasedMapping());
+ this(new RawScriptBasedMapping());
+ }
+
+ /**
+ * Create an instance from the given raw mapping
+ * @param rawMap raw DNSTOSwithMapping
+ */
+ public ScriptBasedMapping(DNSToSwitchMapping rawMap) {
+ super(rawMap);
}
/**
@@ -132,7 +141,7 @@ public final class ScriptBasedMapping ex
* This is the uncached script mapping that is fed into the cache managed
* by the superclass {@link CachedDNSToSwitchMapping}
*/
- private static final class RawScriptBasedMapping
+ protected static class RawScriptBasedMapping
extends AbstractDNSToSwitchMapping {
private String scriptName;
private int maxArgs; //max hostnames per call of the script
@@ -176,7 +185,7 @@ public final class ScriptBasedMapping ex
return m;
}
- String output = runResolveCommand(names);
+ String output = runResolveCommand(names, scriptName);
if (output != null) {
StringTokenizer allSwitchInfo = new StringTokenizer(output);
while (allSwitchInfo.hasMoreTokens()) {
@@ -208,7 +217,8 @@ public final class ScriptBasedMapping ex
* @return null if the number of arguments is out of range,
* or the output of the command.
*/
- private String runResolveCommand(List<String> args) {
+ protected String runResolveCommand(List<String> args,
+ String commandScriptName) {
int loopCount = 0;
if (args.size() == 0) {
return null;
@@ -225,7 +235,7 @@ public final class ScriptBasedMapping ex
while (numProcessed != args.size()) {
int start = maxArgs * loopCount;
List<String> cmdList = new ArrayList<String>();
- cmdList.add(scriptName);
+ cmdList.add(commandScriptName);
for (numProcessed = start; numProcessed < (start + maxArgs) &&
numProcessed < args.size(); numProcessed++) {
cmdList.add(args.get(numProcessed));
Added:
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/ScriptBasedMappingWithDependency.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/ScriptBasedMappingWithDependency.java?rev=1592179&view=auto
==============================================================================
---
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/ScriptBasedMappingWithDependency.java
(added)
+++
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/ScriptBasedMappingWithDependency.java
Sat May 3 11:02:44 2014
@@ -0,0 +1,178 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.net;
+
+import java.util.*;
+import java.util.concurrent.ConcurrentHashMap;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeys;
+
+
+/**
+ * This class extends ScriptBasedMapping class and implements
+ * the {@link DNSToSwitchMappingWithDependency} interface using
+ * a script configured via the
+ * {@link CommonConfigurationKeys#NET_DEPENDENCY_SCRIPT_FILE_NAME_KEY} option.
+ * <p/>
+ * It contains a static class <code>RawScriptBasedMappingWithDependency</code>
+ * that performs the getDependency work.
+ * <p/>
+ */
[email protected]
[email protected]
+public class ScriptBasedMappingWithDependency extends ScriptBasedMapping
+ implements DNSToSwitchMappingWithDependency {
+ /**
+ * key to the dependency script filename {@value}
+ */
+ static final String DEPENDENCY_SCRIPT_FILENAME_KEY =
+ CommonConfigurationKeys.NET_DEPENDENCY_SCRIPT_FILE_NAME_KEY;
+
+ private Map<String, List<String>> dependencyCache =
+ new ConcurrentHashMap<String, List<String>>();
+
+ /**
+ * Create an instance with the default configuration.
+ * </p>
+ * Calling {@link #setConf(Configuration)} will trigger a
+ * re-evaluation of the configuration settings and so be used to
+ * set up the mapping script.
+ */
+ public ScriptBasedMappingWithDependency() {
+ super(new RawScriptBasedMappingWithDependency());
+ }
+
+ /**
+ * Get the cached mapping and convert it to its real type
+ * @return the inner raw script mapping.
+ */
+ private RawScriptBasedMappingWithDependency getRawMapping() {
+ return (RawScriptBasedMappingWithDependency)rawMapping;
+ }
+
+ @Override
+ public String toString() {
+ return "script-based mapping with " + getRawMapping().toString();
+ }
+
+ /**
+ * {@inheritDoc}
+ * <p/>
+ * This will get called in the superclass constructor, so a check is needed
+ * to ensure that the raw mapping is defined before trying to relaying a null
+ * configuration.
+ * @param conf
+ */
+ @Override
+ public void setConf(Configuration conf) {
+ super.setConf(conf);
+ getRawMapping().setConf(conf);
+ }
+
+ /**
+ * Get dependencies in the topology for a given host
+ * @param name - host name for which we are getting dependency
+ * @return a list of hosts dependent on the provided host name
+ */
+ @Override
+ public List<String> getDependency(String name) {
+ //normalize all input names to be in the form of IP addresses
+ name = NetUtils.normalizeHostName(name);
+
+ if (name==null) {
+ return Collections.emptyList();
+ }
+
+ List<String> dependencies = dependencyCache.get(name);
+ if (dependencies == null) {
+ //not cached
+ dependencies = getRawMapping().getDependency(name);
+ if(dependencies != null) {
+ dependencyCache.put(name, dependencies);
+ }
+ }
+
+ return dependencies;
+}
+
+ /**
+ * This is the uncached script mapping that is fed into the cache managed
+ * by the superclass {@link CachedDNSToSwitchMapping}
+ */
+ private static final class RawScriptBasedMappingWithDependency
+ extends ScriptBasedMapping.RawScriptBasedMapping
+ implements DNSToSwitchMappingWithDependency {
+ private String dependencyScriptName;
+
+ /**
+ * Set the configuration and extract the configuration parameters of
interest
+ * @param conf the new configuration
+ */
+ @Override
+ public void setConf (Configuration conf) {
+ super.setConf(conf);
+ if (conf != null) {
+ dependencyScriptName = conf.get(DEPENDENCY_SCRIPT_FILENAME_KEY);
+ } else {
+ dependencyScriptName = null;
+ }
+ }
+
+ /**
+ * Constructor. The mapping is not ready to use until
+ * {@link #setConf(Configuration)} has been called
+ */
+ public RawScriptBasedMappingWithDependency() {}
+
+ @Override
+ public List<String> getDependency(String name) {
+ if (name==null || dependencyScriptName==null) {
+ return Collections.emptyList();
+ }
+
+ List <String> m = new LinkedList<String>();
+ List <String> args = new ArrayList<String>(1);
+ args.add(name);
+
+ String output = runResolveCommand(args,dependencyScriptName);
+ if (output != null) {
+ StringTokenizer allSwitchInfo = new StringTokenizer(output);
+ while (allSwitchInfo.hasMoreTokens()) {
+ String switchInfo = allSwitchInfo.nextToken();
+ m.add(switchInfo);
+ }
+ } else {
+ // an error occurred. return null to signify this.
+ // (exn was already logged in runResolveCommand)
+ return null;
+ }
+
+ return m;
+ }
+
+ @Override
+ public String toString() {
+ return super.toString() + ", " + dependencyScriptName != null ?
+ ("dependency script " + dependencyScriptName) : NO_SCRIPT;
+ }
+ }
+}
Added:
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestScriptBasedMappingWithDependency.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestScriptBasedMappingWithDependency.java?rev=1592179&view=auto
==============================================================================
---
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestScriptBasedMappingWithDependency.java
(added)
+++
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestScriptBasedMappingWithDependency.java
Sat May 3 11:02:44 2014
@@ -0,0 +1,86 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.net;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hadoop.conf.Configuration;
+
+import junit.framework.TestCase;
+import org.junit.Test;
+
+public class TestScriptBasedMappingWithDependency extends TestCase {
+
+
+ public TestScriptBasedMappingWithDependency() {
+
+ }
+
+ @Test
+ public void testNoArgsMeansNoResult() {
+ Configuration conf = new Configuration();
+ conf.setInt(ScriptBasedMapping.SCRIPT_ARG_COUNT_KEY,
+ ScriptBasedMapping.MIN_ALLOWABLE_ARGS - 1);
+ conf.set(ScriptBasedMapping.SCRIPT_FILENAME_KEY, "any-filename-1");
+ conf.set(ScriptBasedMappingWithDependency.DEPENDENCY_SCRIPT_FILENAME_KEY,
+ "any-filename-2");
+ conf.setInt(ScriptBasedMapping.SCRIPT_ARG_COUNT_KEY, 10);
+
+ ScriptBasedMappingWithDependency mapping = createMapping(conf);
+ List<String> names = new ArrayList<String>();
+ names.add("some.machine.name");
+ names.add("other.machine.name");
+ List<String> result = mapping.resolve(names);
+ assertNull("Expected an empty list for resolve", result);
+ result = mapping.getDependency("some.machine.name");
+ assertNull("Expected an empty list for getDependency", result);
+ }
+
+ @Test
+ public void testNoFilenameMeansSingleSwitch() throws Throwable {
+ Configuration conf = new Configuration();
+ ScriptBasedMapping mapping = createMapping(conf);
+ assertTrue("Expected to be single switch", mapping.isSingleSwitch());
+ assertTrue("Expected to be single switch",
+ AbstractDNSToSwitchMapping.isMappingSingleSwitch(mapping));
+ }
+
+ @Test
+ public void testFilenameMeansMultiSwitch() throws Throwable {
+ Configuration conf = new Configuration();
+ conf.set(ScriptBasedMapping.SCRIPT_FILENAME_KEY, "any-filename");
+ ScriptBasedMapping mapping = createMapping(conf);
+ assertFalse("Expected to be multi switch", mapping.isSingleSwitch());
+ mapping.setConf(new Configuration());
+ assertTrue("Expected to be single switch", mapping.isSingleSwitch());
+ }
+
+ @Test
+ public void testNullConfig() throws Throwable {
+ ScriptBasedMapping mapping = createMapping(null);
+ assertTrue("Expected to be single switch", mapping.isSingleSwitch());
+ }
+
+ private ScriptBasedMappingWithDependency createMapping(Configuration conf) {
+ ScriptBasedMappingWithDependency mapping =
+ new ScriptBasedMappingWithDependency();
+ mapping.setConf(conf);
+ return mapping;
+ }
+}
