Author: mattf
Date: Tue Jul 19 23:18:32 2011
New Revision: 1148583
URL: http://svn.apache.org/viewvc?rev=1148583&view=rev
Log:
HADOOP-7314. Add support for throwing UnknownHostException when a host doesn't
resolve. Needed for MAPREDUCE-2489. Contributed by Jeffrey Naisbitt.
Modified:
hadoop/common/branches/branch-0.20-security/CHANGES.txt
hadoop/common/branches/branch-0.20-security/src/core/org/apache/hadoop/net/CachedDNSToSwitchMapping.java
hadoop/common/branches/branch-0.20-security/src/core/org/apache/hadoop/net/DNSToSwitchMapping.java
hadoop/common/branches/branch-0.20-security/src/core/org/apache/hadoop/net/ScriptBasedMapping.java
hadoop/common/branches/branch-0.20-security/src/test/org/apache/hadoop/net/StaticMapping.java
hadoop/common/branches/branch-0.20-security/src/test/org/apache/hadoop/net/TestScriptBasedMapping.java
Modified: hadoop/common/branches/branch-0.20-security/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security/CHANGES.txt?rev=1148583&r1=1148582&r2=1148583&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security/CHANGES.txt (original)
+++ hadoop/common/branches/branch-0.20-security/CHANGES.txt Tue Jul 19 23:18:32
2011
@@ -23,6 +23,9 @@ Release 0.20.205.0 - unreleased
HADOOP-7432. Back-port HADOOP-7110 to 0.20-security: Implement chmod
in NativeIO library. (Sherry Chen via mattf)
+ HADOOP-7314. Add support for throwing UnknownHostException when a host
+ doesn't resolve. Needed for MAPREDUCE-2489. (Jeffrey Naisbitt via mattf)
+
Release 0.20.204.0 - unreleased
NEW FEATURES
Modified:
hadoop/common/branches/branch-0.20-security/src/core/org/apache/hadoop/net/CachedDNSToSwitchMapping.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security/src/core/org/apache/hadoop/net/CachedDNSToSwitchMapping.java?rev=1148583&r1=1148582&r2=1148583&view=diff
==============================================================================
---
hadoop/common/branches/branch-0.20-security/src/core/org/apache/hadoop/net/CachedDNSToSwitchMapping.java
(original)
+++
hadoop/common/branches/branch-0.20-security/src/core/org/apache/hadoop/net/CachedDNSToSwitchMapping.java
Tue Jul 19 23:18:32 2011
@@ -17,6 +17,9 @@
*/
package org.apache.hadoop.net;
+import java.net.InetAddress;
+import java.net.SocketException;
+import java.net.UnknownHostException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
@@ -37,16 +40,11 @@ public class CachedDNSToSwitchMapping im
this.rawMapping = rawMapping;
}
- public List<String> resolve(List<String> names) {
- // normalize all input names to be in the form of IP addresses
- names = NetUtils.normalizeHostNames(names);
-
- List <String> result = new ArrayList<String>(names.size());
- if (names.isEmpty()) {
- return result;
- }
-
-
+
+ /**
+ * Returns the hosts from 'names' that have not been cached previously
+ */
+ private List<String> getUncachedHosts(List<String> names) {
// find out all names without cached resolved location
List<String> unCachedHosts = new ArrayList<String>(names.size());
for (String name : names) {
@@ -54,27 +52,81 @@ public class CachedDNSToSwitchMapping im
unCachedHosts.add(name);
}
}
-
- // Resolve those names
- List<String> rNames = rawMapping.resolve(unCachedHosts);
-
+ return unCachedHosts;
+ }
+
+ /**
+ * Caches the resolved hosts
+ */
+ private void cacheResolvedHosts(List<String> uncachedHosts,
+ List<String> resolvedHosts) {
// Cache the result
- if (rNames != null) {
- for (int i=0; i<unCachedHosts.size(); i++) {
- cache.put(unCachedHosts.get(i), rNames.get(i));
+ if (resolvedHosts != null) {
+ for (int i=0; i<uncachedHosts.size(); i++) {
+ cache.put(uncachedHosts.get(i), resolvedHosts.get(i));
}
}
-
+ }
+
+ /**
+ * Returns the cached resolution of the list of hostnames/addresses.
+ * Returns null if any of the names are not currently in the cache
+ */
+ private List<String> getCachedHosts(List<String> names) {
+ List<String> result = new ArrayList<String>(names.size());
// Construct the result
for (String name : names) {
- //now everything is in the cache
String networkLocation = cache.get(name);
if (networkLocation != null) {
result.add(networkLocation);
- } else { //resolve all or nothing
+ } else {
return null;
}
}
return result;
}
+
+ /**
+ * Resolves host names and adds them to the cache.
+ * Unlike the 'resolve" method, this won't hide UnknownHostExceptions
+ *
+ * @param names to resolve
+ * @return List of resolved names
+ * @throws UnknownHostException if any hosts cannot be resolved
+ */
+ public List<String> resolveValidHosts(List<String> names)
+ throws UnknownHostException {
+ if (names.isEmpty()) {
+ return new ArrayList<String>();
+ }
+ List<String> addresses = new ArrayList<String>(names.size());
+ for (String name : names) {
+ addresses.add(InetAddress.getByName(name).getHostAddress());
+ }
+
+ List<String> uncachedHosts = this.getUncachedHosts(names);
+
+ // Resolve the uncached hosts
+ List<String> resolvedHosts = rawMapping.resolveValidHosts(uncachedHosts);
+ this.cacheResolvedHosts(uncachedHosts, resolvedHosts);
+ return this.getCachedHosts(addresses);
+ }
+
+ public List<String> resolve(List<String> names) {
+ // normalize all input names to be in the form of IP addresses
+ names = NetUtils.normalizeHostNames(names);
+
+ List <String> result = new ArrayList<String>(names.size());
+ if (names.isEmpty()) {
+ return result;
+ }
+
+ List<String> uncachedHosts = this.getUncachedHosts(names);
+
+ // Resolve the uncached hosts
+ List<String> resolvedHosts = rawMapping.resolve(uncachedHosts);
+ this.cacheResolvedHosts(uncachedHosts, resolvedHosts);
+ return this.getCachedHosts(names);
+
+ }
}
Modified:
hadoop/common/branches/branch-0.20-security/src/core/org/apache/hadoop/net/DNSToSwitchMapping.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security/src/core/org/apache/hadoop/net/DNSToSwitchMapping.java?rev=1148583&r1=1148582&r2=1148583&view=diff
==============================================================================
---
hadoop/common/branches/branch-0.20-security/src/core/org/apache/hadoop/net/DNSToSwitchMapping.java
(original)
+++
hadoop/common/branches/branch-0.20-security/src/core/org/apache/hadoop/net/DNSToSwitchMapping.java
Tue Jul 19 23:18:32 2011
@@ -18,6 +18,7 @@
package org.apache.hadoop.net;
import java.util.List;
+import java.net.UnknownHostException;
/**
* An interface that should be implemented to allow pluggable
@@ -39,4 +40,23 @@ public interface DNSToSwitchMapping {
* @return list of resolved network paths
*/
public List<String> resolve(List<String> names);
+
+ /**
+ * Resolves a list of DNS-names/IP-addresses and returns back a list of
+ * switch information (network paths). One-to-one correspondence must be
+ * maintained between the elements in the lists.
+ * Consider an element in the argument list - x.y.com. The switch information
+ * that is returned must be a network path of the form /foo/rack,
+ * where / is the root, and 'foo' is the switch where 'rack' is connected.
+ * Note the hostname/ip-address is not part of the returned path.
+ * The network topology of the cluster would determine the number of
+ * components in the network path. Unlike 'resolve', names must be
+ * resolvable
+ * @param names
+ * @return list of resolved network paths
+ * @throws UnknownHostException if any hosts are not resolvable
+ */
+ public List<String> resolveValidHosts(List<String> names)
+ throws UnknownHostException;
+
}
Modified:
hadoop/common/branches/branch-0.20-security/src/core/org/apache/hadoop/net/ScriptBasedMapping.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security/src/core/org/apache/hadoop/net/ScriptBasedMapping.java?rev=1148583&r1=1148582&r2=1148583&view=diff
==============================================================================
---
hadoop/common/branches/branch-0.20-security/src/core/org/apache/hadoop/net/ScriptBasedMapping.java
(original)
+++
hadoop/common/branches/branch-0.20-security/src/core/org/apache/hadoop/net/ScriptBasedMapping.java
Tue Jul 19 23:18:32 2011
@@ -20,6 +20,7 @@ package org.apache.hadoop.net;
import java.util.*;
import java.io.*;
+import java.net.UnknownHostException;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -115,6 +116,17 @@ implements Configurable
return m;
}
+ public List<String> resolveValidHosts(List<String> names)
+ throws UnknownHostException {
+ List<String> result = this.resolve(names);
+ if (result != null) {
+ return result;
+ } else {
+ throw new UnknownHostException(
+ "Unknown host(s) returned from ScriptBasedMapping");
+ }
+ }
+
private String runResolveCommand(List<String> args) {
int loopCount = 0;
if (args.size() == 0) {
Modified:
hadoop/common/branches/branch-0.20-security/src/test/org/apache/hadoop/net/StaticMapping.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security/src/test/org/apache/hadoop/net/StaticMapping.java?rev=1148583&r1=1148582&r2=1148583&view=diff
==============================================================================
---
hadoop/common/branches/branch-0.20-security/src/test/org/apache/hadoop/net/StaticMapping.java
(original)
+++
hadoop/common/branches/branch-0.20-security/src/test/org/apache/hadoop/net/StaticMapping.java
Tue Jul 19 23:18:32 2011
@@ -18,6 +18,7 @@
package org.apache.hadoop.net;
import java.util.*;
+import java.net.UnknownHostException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
@@ -59,4 +60,19 @@ public class StaticMapping extends Confi
return m;
}
}
+ public List<String> resolveValidHosts(List<String> names)
+ throws UnknownHostException {
+ List<String> m = new ArrayList<String>();
+ synchronized (nameToRackMap) {
+ for (String name : names) {
+ String rackId;
+ if ((rackId = nameToRackMap.get(name)) != null) {
+ m.add(rackId);
+ } else {
+ throw new UnknownHostException(name);
+ }
+ }
+ return m;
+ }
+ }
}
Modified:
hadoop/common/branches/branch-0.20-security/src/test/org/apache/hadoop/net/TestScriptBasedMapping.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security/src/test/org/apache/hadoop/net/TestScriptBasedMapping.java?rev=1148583&r1=1148582&r2=1148583&view=diff
==============================================================================
---
hadoop/common/branches/branch-0.20-security/src/test/org/apache/hadoop/net/TestScriptBasedMapping.java
(original)
+++
hadoop/common/branches/branch-0.20-security/src/test/org/apache/hadoop/net/TestScriptBasedMapping.java
Tue Jul 19 23:18:32 2011
@@ -19,28 +19,67 @@ package org.apache.hadoop.net;
import java.util.ArrayList;
import java.util.List;
+import java.net.UnknownHostException;
import org.apache.hadoop.conf.Configuration;
import junit.framework.TestCase;
public class TestScriptBasedMapping extends TestCase {
+
+ private ScriptBasedMapping mapping;
+ private Configuration conf;
+ private List<String> names;
+
+ public TestScriptBasedMapping() {
+ mapping = new ScriptBasedMapping();
- public void testNoArgsMeansNoResult() {
- ScriptBasedMapping mapping = new ScriptBasedMapping();
-
- Configuration conf = new Configuration();
+ conf = new Configuration();
conf.setInt(ScriptBasedMapping.SCRIPT_ARG_COUNT_KEY,
ScriptBasedMapping.MIN_ALLOWABLE_ARGS - 1);
conf.set(ScriptBasedMapping.SCRIPT_FILENAME_KEY, "any-filename");
- mapping.setConf(conf);
+ mapping.setConf(conf);
+ }
- List<String> names = new ArrayList<String>();
+ public void testNoArgsMeansNoResult() {
+ names = new ArrayList<String>();
names.add("some.machine.name");
names.add("other.machine.name");
-
List<String> result = mapping.resolve(names);
assertNull(result);
}
+
+ public void testResolveValidInvalidHostException() {
+ names = new ArrayList<String>();
+ names.add("1.com"); // Add invalid hostname that doesn't resolve
+ boolean exceptionThrown = false;
+ try {
+ mapping.resolveValidHosts(names);
+ } catch (UnknownHostException e) {
+ exceptionThrown = true;
+ }
+ assertTrue(
+ "resolveValidHosts did not throw UnknownHostException for invalid
host",
+ exceptionThrown);
+ }
+
+ public void testResolveValidHostNoException() {
+ conf.setInt(ScriptBasedMapping.SCRIPT_ARG_COUNT_KEY,
+ ScriptBasedMapping.MIN_ALLOWABLE_ARGS);
+ conf.set(ScriptBasedMapping.SCRIPT_FILENAME_KEY, "echo");
+ mapping.setConf(conf);
+
+ names = new ArrayList<String>();
+ names.add("some.machine.name");
+ names.add("other.machine.name");
+
+ boolean exceptionThrown = false;
+ try {
+ mapping.resolveValidHosts(names);
+ } catch (UnknownHostException e) {
+ exceptionThrown = true;
+ }
+ assertFalse("resolveValidHosts threw Exception for valid host",
exceptionThrown);
+ }
}
