This is an automated email from the ASF dual-hosted git repository.
rnettleton pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/ambari.git
The following commit(s) were added to refs/heads/trunk by this push:
new 8d1f100 [AMBARI-23397] Blueprint deployments for NameNode Federation
Fail (#822)
8d1f100 is described below
commit 8d1f100212de5d49ee848577391a7a1f25e8dadb
Author: rnettleton <[email protected]>
AuthorDate: Thu Mar 29 10:53:26 2018 -0400
[AMBARI-23397] Blueprint deployments for NameNode Federation Fail (#822)
* Initial changes for NameNode Federation support in Blueprints
* Added unit test to verify changes to BlueprintConfigurationProcessor
---
.../internal/BlueprintConfigurationProcessor.java | 6 +-
.../server/topology/ClusterTopologyImpl.java | 4 +-
.../BlueprintConfigurationProcessorTest.java | 133 +++++++++++++++++++++
3 files changed, 138 insertions(+), 5 deletions(-)
diff --git
a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
index 1bb2879..7f1361a 100644
---
a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
+++
b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
@@ -409,8 +409,8 @@ public class BlueprintConfigurationProcessor {
// if the active/stanbdy namenodes are not specified, assign them
automatically
if (! isNameNodeHAInitialActiveNodeSet(clusterProps) && !
isNameNodeHAInitialStandbyNodeSet(clusterProps)) {
Collection<String> nnHosts =
clusterTopology.getHostAssignmentsForComponent("NAMENODE");
- if (nnHosts.size() != 2) {
- throw new ConfigurationTopologyException("NAMENODE HA requires
exactly 2 hosts running NAMENODE but there are: " +
+ if (nnHosts.size() < 2) {
+ throw new ConfigurationTopologyException("NAMENODE HA requires at
least 2 hosts running NAMENODE but there are: " +
nnHosts.size() + " Hosts: " + nnHosts);
}
@@ -1467,7 +1467,7 @@ public class BlueprintConfigurationProcessor {
if (matchingGroupCount == 0 && cardinality.isValidCount(0)) {
return origValue;
} else {
- if (topology.isNameNodeHAEnabled() && isComponentNameNode() &&
(matchingGroupCount == 2)) {
+ if (topology.isNameNodeHAEnabled() && isComponentNameNode() &&
(matchingGroupCount >= 2)) {
// if this is the defaultFS property, it should reflect the
nameservice name,
// rather than a hostname (used in non-HA scenarios)
if (properties.containsKey("core-site") &&
properties.get("core-site").get("fs.defaultFS").equals(origValue)) {
diff --git
a/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterTopologyImpl.java
b/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterTopologyImpl.java
index ab6d852..4860a94 100644
---
a/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterTopologyImpl.java
+++
b/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterTopologyImpl.java
@@ -210,8 +210,8 @@ public class ClusterTopologyImpl implements ClusterTopology
{
if(isNameNodeHAEnabled()){
Collection<String> nnHosts =
getHostAssignmentsForComponent("NAMENODE");
- if (nnHosts.size() != 2) {
- throw new InvalidTopologyException("NAMENODE HA requires exactly 2
hosts running NAMENODE but there are: " +
+ if (nnHosts.size() < 2) {
+ throw new InvalidTopologyException("NAMENODE HA requires at least
2 hosts running NAMENODE but there are: " +
nnHosts.size() + " Hosts: " + nnHosts);
}
Map<String, String> hadoopEnvConfig =
configuration.getFullProperties().get("hadoop-env");
diff --git
a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java
b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java
index d75a962..0d84550 100644
---
a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java
+++
b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java
@@ -5327,6 +5327,139 @@ public class BlueprintConfigurationProcessorTest
extends EasyMockSupport {
}
@Test
+ public void testDoUpdateForClusterWithNameNodeHAEnabledThreeNameNodes()
throws Exception {
+ final String expectedNameService = "mynameservice";
+ final String expectedHostName = "c6401.apache.ambari.org";
+ final String expectedHostNameTwo = "server-two";
+ final String expectedHostNameThree = "server-three";
+ final String expectedPortNum = "808080";
+ final String expectedNodeOne = "nn1";
+ final String expectedNodeTwo = "nn2";
+ final String expectedHostGroupName = "host_group_1";
+
+ Map<String, Map<String, String>> properties = new HashMap<>();
+
+ Map<String, String> hdfsSiteProperties = new HashMap<>();
+ Map<String, String> hbaseSiteProperties = new HashMap<>();
+ Map<String, String> hadoopEnvProperties = new HashMap<>();
+ Map<String, String> coreSiteProperties = new HashMap<>();
+ Map<String, String> accumuloSiteProperties = new HashMap<>();
+
+ properties.put("hdfs-site", hdfsSiteProperties);
+ properties.put("hadoop-env", hadoopEnvProperties);
+ properties.put("core-site", coreSiteProperties);
+ properties.put("hbase-site", hbaseSiteProperties);
+ properties.put("accumulo-site", accumuloSiteProperties);
+
+ // setup hdfs HA config for test
+ hdfsSiteProperties.put("dfs.nameservices", expectedNameService);
+ hdfsSiteProperties.put("dfs.ha.namenodes.mynameservice", expectedNodeOne +
", " + expectedNodeTwo);
+
+ // setup properties that include exported host group information
+ hdfsSiteProperties.put("dfs.namenode.https-address." + expectedNameService
+ "." + expectedNodeOne, createExportedAddress(expectedPortNum,
expectedHostGroupName));
+ hdfsSiteProperties.put("dfs.namenode.https-address." + expectedNameService
+ "." + expectedNodeTwo, createExportedAddress(expectedPortNum,
expectedHostGroupName));
+ hdfsSiteProperties.put("dfs.namenode.http-address." + expectedNameService
+ "." + expectedNodeOne, createExportedAddress(expectedPortNum,
expectedHostGroupName));
+ hdfsSiteProperties.put("dfs.namenode.http-address." + expectedNameService
+ "." + expectedNodeTwo, createExportedAddress(expectedPortNum,
expectedHostGroupName));
+ hdfsSiteProperties.put("dfs.namenode.rpc-address." + expectedNameService +
"." + expectedNodeOne, createExportedAddress(expectedPortNum,
expectedHostGroupName));
+ hdfsSiteProperties.put("dfs.namenode.rpc-address." + expectedNameService +
"." + expectedNodeTwo, createExportedAddress(expectedPortNum,
expectedHostGroupName));
+
+ // add properties that require the SECONDARY_NAMENODE, which
+ // is not included in this test
+ hdfsSiteProperties.put("dfs.secondary.http.address", "localhost:8080");
+ hdfsSiteProperties.put("dfs.namenode.secondary.http-address",
"localhost:8080");
+
+
+ // add properties that are used in non-HA HDFS NameNode settings
+ // to verify that these are eventually removed by the filter
+ hdfsSiteProperties.put("dfs.namenode.http-address", "localhost:8080");
+ hdfsSiteProperties.put("dfs.namenode.https-address", "localhost:8081");
+ hdfsSiteProperties.put("dfs.namenode.rpc-address", "localhost:8082");
+
+ // configure the defaultFS to use the nameservice URL
+ coreSiteProperties.put("fs.defaultFS", "hdfs://" + expectedNameService);
+
+ // configure the hbase rootdir to use the nameservice URL
+ hbaseSiteProperties.put("hbase.rootdir", "hdfs://" + expectedNameService +
"/hbase/test/root/dir");
+
+ // configure the hbase rootdir to use the nameservice URL
+ accumuloSiteProperties.put("instance.volumes", "hdfs://" +
expectedNameService + "/accumulo/test/instance/volumes");
+
+ Configuration clusterConfig = new Configuration(properties,
Collections.emptyMap());
+
+ Collection<String> hgComponents = new HashSet<>();
+ hgComponents.add("NAMENODE");
+ TestHostGroup group1 = new TestHostGroup(expectedHostGroupName,
hgComponents, Collections.singleton(expectedHostName));
+
+ Collection<String> hgComponents2 = new HashSet<>();
+ hgComponents2.add("NAMENODE");
+ TestHostGroup group2 = new TestHostGroup("host-group-2", hgComponents2,
Collections.singleton(expectedHostNameTwo));
+
+ // add third hostgroup with NAMENODE, to simulate HDFS NameNode Federation
+ TestHostGroup group3 = new TestHostGroup("host-group-3",
Collections.singleton("NAMENODE"),
Collections.singleton(expectedHostNameThree));
+
+ Collection<TestHostGroup> hostGroups = new ArrayList<>();
+ hostGroups.add(group1);
+ hostGroups.add(group2);
+ hostGroups.add(group3);
+
+ expect(stack.getCardinality("NAMENODE")).andReturn(new
Cardinality("1-2")).anyTimes();
+ expect(stack.getCardinality("SECONDARY_NAMENODE")).andReturn(new
Cardinality("1")).anyTimes();
+
+ ClusterTopology topology = createClusterTopology(bp, clusterConfig,
hostGroups);
+ BlueprintConfigurationProcessor updater = new
BlueprintConfigurationProcessor(topology);
+
+ Set<String> updatedConfigTypes =
+ updater.doUpdateForClusterCreate();
+
+ // verify that dfs.internal.nameservices was added
+ assertEquals("dfs.internal.nameservices wasn't added",
expectedNameService, hdfsSiteProperties.get("dfs.internal.nameservices"));
+
+ // verify that the expected hostname was substituted for the host group
name in the config
+ assertEquals("HTTPS address HA property not properly exported",
+ expectedHostName + ":" + expectedPortNum,
hdfsSiteProperties.get("dfs.namenode.https-address." + expectedNameService +
"." + expectedNodeOne));
+ assertEquals("HTTPS address HA property not properly exported",
+ expectedHostName + ":" + expectedPortNum,
hdfsSiteProperties.get("dfs.namenode.https-address." + expectedNameService +
"." + expectedNodeTwo));
+
+ assertEquals("HTTPS address HA property not properly exported",
+ expectedHostName + ":" + expectedPortNum,
hdfsSiteProperties.get("dfs.namenode.http-address." + expectedNameService + "."
+ expectedNodeOne));
+ assertEquals("HTTPS address HA property not properly exported",
+ expectedHostName + ":" + expectedPortNum,
hdfsSiteProperties.get("dfs.namenode.http-address." + expectedNameService + "."
+ expectedNodeTwo));
+
+ assertEquals("HTTPS address HA property not properly exported",
+ expectedHostName + ":" + expectedPortNum,
hdfsSiteProperties.get("dfs.namenode.rpc-address." + expectedNameService + "."
+ expectedNodeOne));
+ assertEquals("HTTPS address HA property not properly exported",
+ expectedHostName + ":" + expectedPortNum,
hdfsSiteProperties.get("dfs.namenode.rpc-address." + expectedNameService + "."
+ expectedNodeTwo));
+
+ assertEquals("fs.defaultFS should not be modified by cluster update when
NameNode HA is enabled.",
+ "hdfs://" + expectedNameService, coreSiteProperties.get("fs.defaultFS"));
+
+ assertEquals("hbase.rootdir should not be modified by cluster update when
NameNode HA is enabled.",
+ "hdfs://" + expectedNameService + "/hbase/test/root/dir",
hbaseSiteProperties.get("hbase.rootdir"));
+
+ assertEquals("instance.volumes should not be modified by cluster update
when NameNode HA is enabled.",
+ "hdfs://" + expectedNameService + "/accumulo/test/instance/volumes",
accumuloSiteProperties.get("instance.volumes"));
+
+ // verify that the non-HA properties are filtered out in HA mode
+ assertFalse("dfs.namenode.http-address should have been filtered out of
this HA configuration",
+ hdfsSiteProperties.containsKey("dfs.namenode.http-address"));
+ assertFalse("dfs.namenode.https-address should have been filtered out of
this HA configuration",
+ hdfsSiteProperties.containsKey("dfs.namenode.https-address"));
+ assertFalse("dfs.namenode.rpc-address should have been filtered out of
this HA configuration",
+ hdfsSiteProperties.containsKey("dfs.namenode.rpc-address"));
+
+
+ // verify that correct configuration types were listed as updated in the
returned set
+ assertEquals("Incorrect number of updated config types returned, set = " +
updatedConfigTypes,
+ 3, updatedConfigTypes.size());
+ assertTrue("Expected config type not found in updated set",
+ updatedConfigTypes.contains("cluster-env"));
+ assertTrue("Expected config type not found in updated set",
+ updatedConfigTypes.contains("hdfs-site"));
+ assertTrue("Expected config type not found in updated set",
+ updatedConfigTypes.contains("hadoop-env"));
+ }
+
+ @Test
public void testDoUpdateForClusterWithNameNodeHANotEnabled() throws
Exception {
final String expectedHostName = "c6401.apache.ambari.org";
final String expectedHostNameTwo = "serverTwo";
--
To stop receiving notification emails like this one, please contact
[email protected].
