Author: eli
Date: Sat Feb 4 20:12:51 2012
New Revision: 1240600
URL: http://svn.apache.org/viewvc?rev=1240600&view=rev
Log:
HDFS-2808. HA: haadmin should use namenode ids. Contributed by Eli Collins
Modified:
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/CHANGES.HDFS-1623.txt
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSHAAdmin.java
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCheckPointForSecurityTokens.java
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java
Modified:
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/CHANGES.HDFS-1623.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/CHANGES.HDFS-1623.txt?rev=1240600&r1=1240599&r2=1240600&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/CHANGES.HDFS-1623.txt
(original)
+++
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/CHANGES.HDFS-1623.txt
Sat Feb 4 20:12:51 2012
@@ -166,3 +166,5 @@ HDFS-2874. Edit log should log to shared
HDFS-2890. DFSUtil#getSuffixIDs should skip unset configurations. (atm)
HDFS-2792. Make fsck work. (atm)
+
+HDFS-2808. HA: haadmin should use namenode ids. (eli)
Modified:
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java?rev=1240600&r1=1240599&r2=1240600&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java
(original)
+++
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java
Sat Feb 4 20:12:51 2012
@@ -1042,4 +1042,39 @@ public class DFSUtil {
RPC.setProtocolEngine(conf, protocol, ProtobufRpcEngine.class);
server.addProtocol(RpcKind.RPC_PROTOCOL_BUFFER, protocol, service);
}
+
+ /**
+ * Map a logical namenode ID to its service address. Use the given
+ * nameservice if specified, or the configured one if none is given.
+ *
+ * @param conf Configuration
+ * @param nsId which nameservice nnId is a part of, optional
+ * @param nnId the namenode ID to get the service addr for
+ * @return the service addr, null if it could not be determined
+ */
+ public static String getNamenodeServiceAddr(final Configuration conf,
+ String nsId, String nnId) {
+
+ if (nsId == null) {
+ Collection<String> nsIds = getNameServiceIds(conf);
+ if (nsIds.size() != 1) {
+ // No nameservice ID was given and more than one is configured
+ return null;
+ } else {
+ nsId = nsIds.toArray(new String[1])[0];
+ }
+ }
+
+ String serviceAddrKey = concatSuffixes(
+ DFSConfigKeys.DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY, nsId, nnId);
+
+ String addrKey = concatSuffixes(
+ DFSConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY, nsId, nnId);
+
+ String serviceRpcAddr = conf.get(serviceAddrKey);
+ if (serviceRpcAddr == null) {
+ serviceRpcAddr = conf.get(addrKey);
+ }
+ return serviceRpcAddr;
+ }
}
Modified:
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSHAAdmin.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSHAAdmin.java?rev=1240600&r1=1240599&r2=1240600&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSHAAdmin.java
(original)
+++
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSHAAdmin.java
Sat Feb 4 20:12:51 2012
@@ -17,12 +17,16 @@
*/
package org.apache.hadoop.hdfs.tools;
+import java.io.PrintStream;
+import java.util.Arrays;
+
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.ha.HAAdmin;
import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.util.ToolRunner;
@@ -30,10 +34,15 @@ import org.apache.hadoop.util.ToolRunner
* Class to extend HAAdmin to do a little bit of HDFS-specific configuration.
*/
public class DFSHAAdmin extends HAAdmin {
-
- private static final Log LOG =
- LogFactory.getLog(DFSHAAdmin.class);
-
+
+ private static final Log LOG = LogFactory.getLog(DFSHAAdmin.class);
+
+ private String nameserviceId;
+
+ protected void setErrOut(PrintStream errOut) {
+ this.errOut = errOut;
+ }
+
@Override
public void setConf(Configuration conf) {
if (conf != null) {
@@ -52,6 +61,54 @@ public class DFSHAAdmin extends HAAdmin
super.setConf(conf);
}
+ /**
+ * Try to map the given namenode ID to its service address.
+ */
+ @Override
+ protected String getServiceAddr(String nnId) {
+ HdfsConfiguration conf = (HdfsConfiguration)getConf();
+ String serviceAddr =
+ DFSUtil.getNamenodeServiceAddr(conf, nameserviceId, nnId);
+ if (serviceAddr == null) {
+ throw new IllegalArgumentException(
+ "Unable to determine service address for namenode '" + nnId + "'");
+ }
+ return serviceAddr;
+ }
+
+ @Override
+ protected String getUsageString() {
+ return "Usage: DFSHAAdmin [-ns <nameserviceId>]";
+ }
+
+ @Override
+ protected int runCmd(String[] argv) throws Exception {
+ if (argv.length < 1) {
+ printUsage(errOut);
+ return -1;
+ }
+
+ int i = 0;
+ String cmd = argv[i++];
+
+ if ("-ns".equals(cmd)) {
+ if (i == argv.length) {
+ errOut.println("Missing nameservice ID");
+ printUsage(errOut);
+ return -1;
+ }
+ nameserviceId = argv[i++];
+ if (i >= argv.length) {
+ errOut.println("Missing command");
+ printUsage(errOut);
+ return -1;
+ }
+ argv = Arrays.copyOfRange(argv, i, argv.length);
+ }
+
+ return super.runCmd(argv);
+ }
+
public static void main(String[] argv) throws Exception {
int res = ToolRunner.run(new DFSHAAdmin(), argv);
System.exit(res);
Modified:
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java?rev=1240600&r1=1240599&r2=1240600&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java
(original)
+++
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java
Sat Feb 4 20:12:51 2012
@@ -406,6 +406,53 @@ public class TestDFSUtil {
assertEquals(NS1_NN2_HOST, map.get("ns1").get("ns1-nn2").toString());
assertEquals(NS2_NN1_HOST, map.get("ns2").get("ns2-nn1").toString());
assertEquals(NS2_NN2_HOST, map.get("ns2").get("ns2-nn2").toString());
+
+ assertEquals(NS1_NN1_HOST,
+ DFSUtil.getNamenodeServiceAddr(conf, "ns1", "ns1-nn1"));
+ assertEquals(NS1_NN2_HOST,
+ DFSUtil.getNamenodeServiceAddr(conf, "ns1", "ns1-nn2"));
+ assertEquals(NS2_NN1_HOST,
+ DFSUtil.getNamenodeServiceAddr(conf, "ns2", "ns2-nn1"));
+
+ // No nameservice was given and we can't determine which to use
+ // as two nameservices could share a namenode ID.
+ assertEquals(null, DFSUtil.getNamenodeServiceAddr(conf, null, "ns1-nn1"));
+ }
+
+ @Test
+ public void getNameNodeServiceAddr() throws IOException {
+ HdfsConfiguration conf = new HdfsConfiguration();
+
+ // One nameservice with two NNs
+ final String NS1_NN1_HOST = "ns1-nn1.example.com:8020";
+ final String NS1_NN1_HOST_SVC = "ns1-nn2.example.com:8021";
+ final String NS1_NN2_HOST = "ns1-nn1.example.com:8020";
+ final String NS1_NN2_HOST_SVC = "ns1-nn2.example.com:8021";
+
+ conf.set(DFS_FEDERATION_NAMESERVICES, "ns1");
+ conf.set(DFSUtil.addKeySuffixes(DFS_HA_NAMENODES_KEY, "ns1"),"nn1,nn2");
+
+ conf.set(DFSUtil.addKeySuffixes(
+ DFS_NAMENODE_RPC_ADDRESS_KEY, "ns1", "nn1"), NS1_NN1_HOST);
+ conf.set(DFSUtil.addKeySuffixes(
+ DFS_NAMENODE_RPC_ADDRESS_KEY, "ns1", "nn2"), NS1_NN2_HOST);
+
+ // The rpc address is used if no service address is defined
+ assertEquals(NS1_NN1_HOST, DFSUtil.getNamenodeServiceAddr(conf, null,
"nn1"));
+ assertEquals(NS1_NN2_HOST, DFSUtil.getNamenodeServiceAddr(conf, null,
"nn2"));
+
+ // A nameservice is specified explicitly
+ assertEquals(NS1_NN1_HOST, DFSUtil.getNamenodeServiceAddr(conf, "ns1",
"nn1"));
+ assertEquals(null, DFSUtil.getNamenodeServiceAddr(conf, "invalid", "nn1"));
+
+ // The service addrs are used when they are defined
+ conf.set(DFSUtil.addKeySuffixes(
+ DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY, "ns1", "nn1"), NS1_NN1_HOST_SVC);
+ conf.set(DFSUtil.addKeySuffixes(
+ DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY, "ns1", "nn2"), NS1_NN2_HOST_SVC);
+
+ assertEquals(NS1_NN1_HOST_SVC, DFSUtil.getNamenodeServiceAddr(conf, null,
"nn1"));
+ assertEquals(NS1_NN2_HOST_SVC, DFSUtil.getNamenodeServiceAddr(conf, null,
"nn2"));
}
@Test
Modified:
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCheckPointForSecurityTokens.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCheckPointForSecurityTokens.java?rev=1240600&r1=1240599&r2=1240600&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCheckPointForSecurityTokens.java
(original)
+++
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCheckPointForSecurityTokens.java
Sat Feb 4 20:12:51 2012
@@ -57,7 +57,7 @@ public class TestCheckPointForSecurityTo
}
/**
- * Tests save namepsace.
+ * Tests save namespace.
*/
@Test
public void testSaveNamespace() throws IOException {
Modified:
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java?rev=1240600&r1=1240599&r2=1240600&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java
(original)
+++
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java
Sat Feb 4 20:12:51 2012
@@ -861,7 +861,7 @@ public class TestCheckpoint extends Test
}
/**
- * Tests save namepsace.
+ * Tests save namespace.
*/
public void testSaveNamespace() throws IOException {
MiniDFSCluster cluster = null;
