Repository: hadoop Updated Branches: refs/heads/branch-2 43131af1e -> 80dd039b6 refs/heads/branch-2.8 aeea77ce1 -> 94950c15c refs/heads/trunk db144eb1c -> 4d7055e0c
HDFS-9718. HAUtil#getConfForOtherNodes should unset independent generic keys before initialize (Contributed by DENG FEI) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4d7055e0 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4d7055e0 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4d7055e0 Branch: refs/heads/trunk Commit: 4d7055e0c8d1936d6c87cec14dbbfdff77b837f3 Parents: db144eb Author: Vinayakumar B <[email protected]> Authored: Tue Feb 2 12:51:35 2016 +0530 Committer: Vinayakumar B <[email protected]> Committed: Tue Feb 2 12:51:35 2016 +0530 ---------------------------------------------------------------------- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ .../java/org/apache/hadoop/hdfs/HAUtil.java | 27 ++++++++++++++++++++ .../server/namenode/ha/TestHAConfiguration.java | 23 +++++++++++++++++ 3 files changed, 53 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/4d7055e0/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index f77451b..366f330 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -2672,6 +2672,9 @@ Release 2.8.0 - UNRELEASED HDFS-9701. DN may deadlock when hot-swapping under load. (Xiao Chen via lei) + HDFS-9718. HAUtil#getConfForOtherNodes should unset independent generic keys + before initialize (DENG FEI via vinayakumarb) + Release 2.7.3 - UNRELEASED INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/4d7055e0/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HAUtil.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HAUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HAUtil.java index ff409c3..7b65abf 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HAUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HAUtil.java @@ -18,7 +18,16 @@ package org.apache.hadoop.hdfs; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HA_NAMENODE_ID_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_HTTPS_ADDRESS_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_HTTPS_BIND_HOST_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_HTTP_BIND_HOST_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_LIFELINE_RPC_ADDRESS_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_LIFELINE_RPC_BIND_HOST_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_RPC_BIND_HOST_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_SERVICE_RPC_BIND_HOST_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_SHARED_EDITS_DIR_KEY; import java.io.IOException; @@ -64,6 +73,19 @@ public class HAUtil { private static final DelegationTokenSelector tokenSelector = new DelegationTokenSelector(); + private static final String[] HA_SPECIAL_INDEPENDENT_KEYS = new String[]{ + DFS_NAMENODE_RPC_ADDRESS_KEY, + DFS_NAMENODE_RPC_BIND_HOST_KEY, + DFS_NAMENODE_LIFELINE_RPC_ADDRESS_KEY, + DFS_NAMENODE_LIFELINE_RPC_BIND_HOST_KEY, + DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY, + DFS_NAMENODE_SERVICE_RPC_BIND_HOST_KEY, + DFS_NAMENODE_HTTP_ADDRESS_KEY, + DFS_NAMENODE_HTTPS_ADDRESS_KEY, + DFS_NAMENODE_HTTP_BIND_HOST_KEY, + DFS_NAMENODE_HTTPS_BIND_HOST_KEY, + }; + private HAUtil() { /* Hidden constructor */ } /** @@ -188,6 +210,11 @@ public class HAUtil { // Look up the address of the other NNs List<Configuration> confs = new ArrayList<Configuration>(otherNn.size()); + myConf = new Configuration(myConf); + // unset independent properties + for (String idpKey : HA_SPECIAL_INDEPENDENT_KEYS) { + myConf.unset(idpKey); + } for (String nn : otherNn) { Configuration confForOtherNode = new Configuration(myConf); NameNode.initializeGenericKeys(confForOtherNode, nsId, nn); http://git-wip-us.apache.org/repos/asf/hadoop/blob/4d7055e0/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHAConfiguration.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHAConfiguration.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHAConfiguration.java index 62643ae..b383372 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHAConfiguration.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHAConfiguration.java @@ -20,6 +20,8 @@ package org.apache.hadoop.hdfs.server.namenode.ha; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_EDITS_DIR_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_SHARED_EDITS_DIR_KEY; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import java.io.IOException; @@ -27,11 +29,13 @@ import java.net.MalformedURLException; import java.net.URI; import java.net.URL; import java.util.Collection; +import java.util.List; import com.google.common.base.Joiner; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DFSUtil; +import org.apache.hadoop.hdfs.HAUtil; import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; import org.apache.hadoop.hdfs.server.namenode.NameNode; import org.apache.hadoop.hdfs.server.namenode.SecondaryNameNode; @@ -150,4 +154,23 @@ public class TestHAConfiguration { "Cannot use SecondaryNameNode in an HA cluster", ioe); } } + + @Test + public void testGetOtherNNGenericConf() throws IOException { + String nsId = "ns1"; + String host1 = "1.2.3.1"; + String host2 = "1.2.3.2"; + Configuration conf = getHAConf(nsId, host1, host2); + conf.set(DFSUtil.addKeySuffixes( + DFSConfigKeys.DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY, nsId, "nn1"), + host1 + ":54321"); + conf.set(DFSConfigKeys.DFS_NAMESERVICE_ID, "ns1"); + NameNode.initializeGenericKeys(conf, "ns1", "nn1"); + List<Configuration> others = HAUtil.getConfForOtherNodes(conf); + Configuration nn2Conf = others.get(0); + assertEquals(nn2Conf.get(DFSConfigKeys.DFS_HA_NAMENODE_ID_KEY),"nn2"); + assertTrue(!conf.get(DFSConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY) + .equals(nn2Conf.get(DFSConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY))); + assertNull(nn2Conf.get(DFSConfigKeys.DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY)); + } }
