Author: kihwal
Date: Fri May 9 02:03:27 2014
New Revision: 1593476
URL: http://svn.apache.org/r1593476
Log:
svn merge -c 1593475 merging from trunk to branch-2 to fix:HDFS-6313. WebHdfs
may use the wrong NN when configured for multiple HA NNs. Contributed by Kihwal
Lee.
Modified:
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFSForHA.java
Modified:
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1593476&r1=1593475&r2=1593476&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
(original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
Fri May 9 02:03:27 2014
@@ -249,6 +249,9 @@ Release 2.4.1 - UNRELEASED
HDFS-6329. WebHdfs does not work if HA is enabled on NN but logical URI is
not configured. (kihwal)
+ HDFS-6313. WebHdfs may use the wrong NN when configured for multiple HA NNs
+ (kihwal)
+
Release 2.4.0 - 2014-04-07
INCOMPATIBLE CHANGES
Modified:
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java?rev=1593476&r1=1593475&r2=1593476&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
Fri May 9 02:03:27 2014
@@ -101,6 +101,7 @@ import org.apache.hadoop.security.token.
import org.apache.hadoop.util.Progressable;
import org.mortbay.util.ajax.JSON;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Charsets;
import com.google.common.collect.Lists;
@@ -1112,10 +1113,10 @@ public class WebHdfsFileSystem extends F
Map<String, Map<String, InetSocketAddress>> addresses = DFSUtil
.getHaNnWebHdfsAddresses(conf, scheme);
- for (Map<String, InetSocketAddress> addrs : addresses.values()) {
- for (InetSocketAddress addr : addrs.values()) {
- ret.add(addr);
- }
+ // Extract the entry corresponding to the logical name.
+ Map<String, InetSocketAddress> addrs = addresses.get(uri.getHost());
+ for (InetSocketAddress addr : addrs.values()) {
+ ret.add(addr);
}
}
@@ -1128,4 +1129,9 @@ public class WebHdfsFileSystem extends F
return tokenServiceName == null ? super.getCanonicalServiceName()
: tokenServiceName.toString();
}
+
+ @VisibleForTesting
+ InetSocketAddress[] getResolvedNNAddr() {
+ return nnAddrs;
+ }
}
Modified:
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java?rev=1593476&r1=1593475&r2=1593476&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
Fri May 9 02:03:27 2014
@@ -148,16 +148,40 @@ public class DFSTestUtil {
*/
public static Configuration newHAConfiguration(final String logicalName) {
Configuration conf = new Configuration();
- conf.set(DFSConfigKeys.DFS_NAMESERVICES, logicalName);
+ addHAConfiguration(conf, logicalName);
+ return conf;
+ }
+
+ /**
+ * Add a new HA configuration.
+ */
+ public static void addHAConfiguration(Configuration conf,
+ final String logicalName) {
+ String nsIds = conf.get(DFSConfigKeys.DFS_NAMESERVICES);
+ if (nsIds == null) {
+ conf.set(DFSConfigKeys.DFS_NAMESERVICES, logicalName);
+ } else { // append the nsid
+ conf.set(DFSConfigKeys.DFS_NAMESERVICES, nsIds + "," + logicalName);
+ }
conf.set(DFSUtil.addKeySuffixes(DFSConfigKeys.DFS_HA_NAMENODES_KEY_PREFIX,
logicalName), "nn1,nn2");
conf.set(DFSConfigKeys.DFS_CLIENT_FAILOVER_PROXY_PROVIDER_KEY_PREFIX + "" +
"." + logicalName,
ConfiguredFailoverProxyProvider.class.getName());
conf.setInt(DFSConfigKeys.DFS_REPLICATION_KEY, 1);
- return conf;
}
+ public static void setFakeHttpAddresses(Configuration conf,
+ final String logicalName) {
+ conf.set(DFSUtil.addKeySuffixes(
+ DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_KEY,
+ logicalName, "nn1"), "127.0.0.1:12345");
+ conf.set(DFSUtil.addKeySuffixes(
+ DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_KEY,
+ logicalName, "nn2"), "127.0.0.1:12346");
+ }
+
+
/** class MyFile contains enough information to recreate the contents of
* a single file.
*/
Modified:
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFSForHA.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFSForHA.java?rev=1593476&r1=1593475&r2=1593476&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFSForHA.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFSForHA.java
Fri May 9 02:03:27 2014
@@ -156,4 +156,30 @@ public class TestWebHDFSForHA {
}
}
}
+
+ @Test
+ public void testMultipleNamespacesConfigured() throws Exception {
+ Configuration conf = DFSTestUtil.newHAConfiguration(LOGICAL_NAME);
+ MiniDFSCluster cluster = null;
+ WebHdfsFileSystem fs = null;
+
+ try {
+ cluster = new MiniDFSCluster.Builder(conf).nnTopology(topo)
+ .numDataNodes(1).build();
+
+ HATestUtil.setFailoverConfigurations(cluster, conf, LOGICAL_NAME);
+
+ cluster.waitActive();
+ DFSTestUtil.addHAConfiguration(conf, LOGICAL_NAME + "remote");
+ DFSTestUtil.setFakeHttpAddresses(conf, LOGICAL_NAME + "remote");
+
+ fs = (WebHdfsFileSystem)FileSystem.get(WEBHDFS_URI, conf);
+ Assert.assertEquals(2, fs.getResolvedNNAddr().length);
+ } finally {
+ IOUtils.cleanup(null, fs);
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+ }
}
