This is an automated email from the ASF dual-hosted git repository.
vishalsuvagia pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/ambari.git
The following commit(s) were added to refs/heads/trunk by this push:
new e2ce601 [AMBARI-23864] : Need to add new property for Ranger-Tagsync
when enabl… (#1291)
e2ce601 is described below
commit e2ce601032d2dd1f047793aa51d749749017774b
Author: Vishal Suvagia <[email protected]>
AuthorDate: Thu May 24 11:51:08 2018 +0530
[AMBARI-23864] : Need to add new property for Ranger-Tagsync when enabl…
(#1291)
* AMBARI-23864 : Need to add new property for Ranger-Tagsync when enabling
federation for Namenode-HA via Blueprints
* AMBARI-23864 : Need to add new property for Ranger-Tagsync when enabling
federation for Namenode-HA via Blueprints
* AMBARI-23864 : Need to add new property for Ranger-Tagsync when enabling
federation for Namenode-HA via Blueprints
* AMBARI-23864 : Need to add new property for Ranger-Tagsync when enabling
federation for Namenode-HA via Blueprints
---
.../internal/BlueprintConfigurationProcessor.java | 70 ++
.../BlueprintConfigurationProcessorTest.java | 705 ++++++++++++++++++++-
2 files changed, 774 insertions(+), 1 deletion(-)
diff --git
a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
index 24dcceb..06bac12 100644
---
a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
+++
b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
@@ -22,6 +22,8 @@ package org.apache.ambari.server.controller.internal;
import static java.util.stream.Collectors.groupingBy;
import static java.util.stream.Collectors.toSet;
+import java.net.URI;
+import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
@@ -114,6 +116,7 @@ public class BlueprintConfigurationProcessor {
);
private final static String HADOOP_ENV_CONFIG_TYPE_NAME = "hadoop-env";
+ private final static String RANGER_TAGSYNC_SITE_CONFIG_TYPE_NAME =
"ranger-tagsync-site";
/**
@@ -542,12 +545,79 @@ public class BlueprintConfigurationProcessor {
} else {
LOG.warn("Error in processing the set of active/standby namenodes
in this federated cluster, please check hdfs-site configuration");
}
+ // Need to configure ranger-tagsync-site properties, when
Ranger-HDFS plugin is enabled
+
doTagSyncSiteUpdateForNamenodeNFederationEnabledOnClusterCreation(clusterConfig,
clusterProps, configTypesUpdated);
}
}
}
}
/**
+ * Update ranger-tagsync-site properties when NN Federation is enabled and
Ranger-HDFS plugin is enabled
+ * @throws ConfigurationTopologyException
+ */
+ private void
doTagSyncSiteUpdateForNamenodeNFederationEnabledOnClusterCreation(Configuration
clusterConfig,
+
Map<String, Map<String, String>> clusterProps,
+
Set<String> configTypesUpdated) throws ConfigurationTopologyException {
+ Map<String, String> hdfsSiteConfig =
clusterConfig.getFullProperties().get("hdfs-site");
+ // parse out the nameservices value
+ String[] parsedNameServices = parseNameServices(hdfsSiteConfig);
+ String clusterName = getClusterName();
+
+ // Getting configuration and properties for adding configurations to
ranger-tagsync-site
+ boolean isRangerHDFSPluginEnabled = false;
+ String rangerHDFSPluginServiceName = "";
+
+ String atlasServerComponentName = "ATLAS_SERVER";
+ String rangerAdminComponentName = "RANGER_ADMIN";
+ String rangerTagsyncComponentName = "RANGER_TAGSYNC";
+ boolean isRangerAdminToBeInstalled =
(clusterTopology.getHostGroupsForComponent(rangerAdminComponentName).size() >=
1);
+ boolean isRangerTagsyncToBeInstalled =
(clusterTopology.getHostGroupsForComponent(rangerTagsyncComponentName).size()
>= 1);
+ boolean isAtlasServerToBeInstalled =
(clusterTopology.getHostGroupsForComponent(atlasServerComponentName).size() >=
1);
+ if (isRangerAdminToBeInstalled) {
+ Map<String, String> rangerHDFSPluginProperties =
clusterProps.get("ranger-hdfs-plugin-properties");
+ String rangerHDFSPluginEnabledValue =
rangerHDFSPluginProperties.getOrDefault("ranger-hdfs-plugin-enabled","No");
+ isRangerHDFSPluginEnabled =
("yes".equalsIgnoreCase(rangerHDFSPluginEnabledValue));
+ Map<String, String> rangerHDFSSecurityConfig =
clusterProps.get("ranger-hdfs-security");
+ rangerHDFSPluginServiceName =
rangerHDFSSecurityConfig.get("ranger.plugin.hdfs.service.name");
+ }
+
+ boolean isTagsyncPropertyConfigurationRequired = (
isRangerAdminToBeInstalled && isRangerTagsyncToBeInstalled &&
+
isAtlasServerToBeInstalled && isRangerHDFSPluginEnabled );
+
+ Map<String, String> coreSiteConfig = clusterProps.get("core-site");
+ String fsDefaultFSValue = coreSiteConfig.get("fs.defaultFS");
+ String nameServiceInFsDefaultFSConfig="";
+
+ if (isTagsyncPropertyConfigurationRequired &&
"{{repo_name}}".equalsIgnoreCase(rangerHDFSPluginServiceName)) {
+ rangerHDFSPluginServiceName = clusterName + "_hadoop";
+ }
+ // If blueprint configuration has multiple nameservices, indicating
NN-Federation is enabled
+ if (parsedNameServices.length > 1 &&
isTagsyncPropertyConfigurationRequired) {
+ for (String nameService : parsedNameServices) {
+ // Adding configurations for Ranger-Tagsync to map Ranger HDFS service
for Atlas Tagsync
+ String tagsyncNameserviceMappingProperty =
"ranger.tagsync.atlas.hdfs.instance." + clusterName + ".nameservice." +
nameService + ".ranger.service";
+ String updatedRangerHDFSPluginServiceName =
rangerHDFSPluginServiceName + "_" + nameService;
+ clusterConfig.setProperty(RANGER_TAGSYNC_SITE_CONFIG_TYPE_NAME,
tagsyncNameserviceMappingProperty, updatedRangerHDFSPluginServiceName);
+ try {
+ URI fsDefaultFSURI = new URI(fsDefaultFSValue);
+ String fsDefaultFSNameService = fsDefaultFSURI.getHost();
+ if (fsDefaultFSNameService.contains(nameService)) {
+ nameServiceInFsDefaultFSConfig = nameService;
+ }
+ } catch (URISyntaxException e) {
+ LOG.error("Error occurred while parsing the defaultFS URI.", e);
+ }
+ }
+
+ String rangerTagsyncAtlasNNServiceMappingProperty =
"ranger.tagsync.atlas.hdfs.instance." + clusterName + ".ranger.service";
+ String rangerTagsyncAtlasNNServiceName = rangerHDFSPluginServiceName +
"_" + nameServiceInFsDefaultFSConfig;
+ clusterConfig.setProperty(RANGER_TAGSYNC_SITE_CONFIG_TYPE_NAME,
rangerTagsyncAtlasNNServiceMappingProperty, rangerTagsyncAtlasNNServiceName);
+ configTypesUpdated.add(RANGER_TAGSYNC_SITE_CONFIG_TYPE_NAME);
+ }
+ }
+
+ /**
* Call registered updaters on cluster configuration + call default updater
({@link HostGroupUpdater#INSTANCE}) on
* properties that were submitted in the blueprint or the cluster template
and don't have a registered updater.
*/
diff --git
a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java
b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java
index 54b9442..b5bc796 100644
---
a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java
+++
b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java
@@ -233,6 +233,12 @@ public class BlueprintConfigurationProcessorTest extends
EasyMockSupport {
stormComponents.add("NIMBUS");
serviceComponents.put("STORM", stormComponents);
+ Collection<String> rangerComponents = new HashSet<>();
+ rangerComponents.add("RANGER_ADMIN");
+ rangerComponents.add("RANGER_USERSYNC");
+ rangerComponents.add("RANGER_TAGSYNC");
+ serviceComponents.put("RANGER",rangerComponents);
+
for (Map.Entry<String, Collection<String>> entry :
serviceComponents.entrySet()) {
String service = entry.getKey();
for (String component : entry.getValue()) {
@@ -248,7 +254,7 @@ public class BlueprintConfigurationProcessorTest extends
EasyMockSupport {
expect(ambariContext.getConfigHelper()).andReturn(configHelper).anyTimes();
expect(configHelper.getDefaultStackProperties(
EasyMock.eq(new StackId(STACK_NAME,
STACK_VERSION)))).andReturn(stackProperties).anyTimes();
-
+
stackProperties.put(ConfigHelper.CLUSTER_ENV, defaultClusterEnvProperties);
@@ -9152,6 +9158,703 @@ public class BlueprintConfigurationProcessorTest
extends EasyMockSupport {
clusterConfig.getPropertyValue("hdfs-site", "test.single.space"));
}
+ @Test
+ public void
testDoUpdateForClusterWithNameNodeFederationEnabledTagsyncEnabledDefaultRepoName()
throws Exception {
+ final String expectedNameService = "mynameservice";
+ final String expectedNameServiceTwo = "mynameservicetwo";
+ final String expectedHostName = "c6401.apache.ambari.org";
+ final String expectedHostNameTwo = "c6402.apache.ambari.org";
+ final String expectedHostNameThree = "c6403.apache.ambari.org";
+ final String expectedHostNameFour = "c6404.apache.ambari.org";
+ final String expectedPortNum = "808080";
+ final String expectedNodeOne = "nn1";
+ final String expectedNodeTwo = "nn2";
+ final String expectedNodeThree = "nn3";
+ final String expectedNodeFour = "nn4";
+ final String expectedHostGroupName = "host_group_1";
+ final String expectedHostGroupNameTwo = "host_group_2";
+ final String expectedHostGroupNameThree = "host-group-3";
+ final String expectedHostGroupNameFour = "host-group-4";
+
+ Map<String, Map<String, String>> properties = new HashMap<>();
+
+ Map<String, String> hdfsSiteProperties = new HashMap<>();
+ Map<String, String> hbaseSiteProperties = new HashMap<>();
+ Map<String, String> hadoopEnvProperties = new HashMap<>();
+ Map<String, String> coreSiteProperties = new HashMap<>();
+ Map<String, String> accumuloSiteProperties = new HashMap<>();
+ Map<String, String> rangerHDFSPluginProperties = new HashMap<>();
+ Map<String, String> rangerHDFSSecurityProperties = new HashMap<>();
+
+ properties.put("hdfs-site", hdfsSiteProperties);
+ properties.put("hadoop-env", hadoopEnvProperties);
+ properties.put("core-site", coreSiteProperties);
+ properties.put("hbase-site", hbaseSiteProperties);
+ properties.put("accumulo-site", accumuloSiteProperties);
+ properties.put("ranger-hdfs-plugin-properties",
rangerHDFSPluginProperties);
+ properties.put("ranger-hdfs-security", rangerHDFSSecurityProperties);
+
+ // setup multiple nameservices, to indicate NameNode Federation will be
used
+ hdfsSiteProperties.put("dfs.nameservices", expectedNameService + "," +
expectedNameServiceTwo);
+ hdfsSiteProperties.put("dfs.ha.namenodes.mynameservice", expectedNodeOne +
", " + expectedNodeTwo);
+ hdfsSiteProperties.put("dfs.ha.namenodes." + expectedNameServiceTwo,
expectedNodeThree + "," + expectedNodeFour);
+
+ //setup nameservice-specific properties
+ hdfsSiteProperties.put("dfs.namenode.shared.edits.dir" + "." +
expectedNameService,
+ "qjournal://" + createExportedAddress(expectedPortNum,
expectedHostGroupName) + ";" + createExportedAddress(expectedPortNum,
expectedHostGroupNameTwo) + "/ns1");
+ hdfsSiteProperties.put("dfs.namenode.shared.edits.dir" + "." +
expectedNameServiceTwo,
+ "qjournal://" + createExportedAddress(expectedPortNum,
expectedHostGroupName) + ";" + createExportedAddress(expectedPortNum,
expectedHostGroupNameTwo) + "/ns2");
+
+
+ // setup properties that include exported host group information
+ hdfsSiteProperties.put("dfs.namenode.https-address." + expectedNameService
+ "." + expectedNodeOne, createExportedAddress(expectedPortNum,
expectedHostGroupName));
+ hdfsSiteProperties.put("dfs.namenode.https-address." + expectedNameService
+ "." + expectedNodeTwo, createExportedAddress(expectedPortNum,
expectedHostGroupNameTwo));
+ hdfsSiteProperties.put("dfs.namenode.http-address." + expectedNameService
+ "." + expectedNodeOne, createExportedAddress(expectedPortNum,
expectedHostGroupName));
+ hdfsSiteProperties.put("dfs.namenode.http-address." + expectedNameService
+ "." + expectedNodeTwo, createExportedAddress(expectedPortNum,
expectedHostGroupNameTwo));
+ hdfsSiteProperties.put("dfs.namenode.rpc-address." + expectedNameService +
"." + expectedNodeOne, createExportedAddress(expectedPortNum,
expectedHostGroupName));
+ hdfsSiteProperties.put("dfs.namenode.rpc-address." + expectedNameService +
"." + expectedNodeTwo, createExportedAddress(expectedPortNum,
expectedHostGroupNameTwo));
+ hdfsSiteProperties.put("dfs.namenode.servicerpc-address." +
expectedNameService + "." + expectedNodeOne,
createExportedAddress(expectedPortNum, expectedHostGroupName));
+ hdfsSiteProperties.put("dfs.namenode.servicerpc-address." +
expectedNameService + "." + expectedNodeTwo,
createExportedAddress(expectedPortNum, expectedHostGroupNameTwo));
+
+ hdfsSiteProperties.put("dfs.namenode.https-address." +
expectedNameServiceTwo + "." + expectedNodeThree,
createExportedAddress(expectedPortNum, expectedHostGroupNameThree));
+ hdfsSiteProperties.put("dfs.namenode.https-address." +
expectedNameServiceTwo + "." + expectedNodeFour,
createExportedAddress(expectedPortNum, expectedHostGroupNameFour));
+ hdfsSiteProperties.put("dfs.namenode.http-address." +
expectedNameServiceTwo + "." + expectedNodeThree,
createExportedAddress(expectedPortNum, expectedHostGroupNameThree));
+ hdfsSiteProperties.put("dfs.namenode.http-address." +
expectedNameServiceTwo + "." + expectedNodeFour,
createExportedAddress(expectedPortNum, expectedHostGroupNameFour));
+ hdfsSiteProperties.put("dfs.namenode.rpc-address." +
expectedNameServiceTwo + "." + expectedNodeThree,
createExportedAddress(expectedPortNum, expectedHostGroupNameThree));
+ hdfsSiteProperties.put("dfs.namenode.rpc-address." +
expectedNameServiceTwo + "." + expectedNodeFour,
createExportedAddress(expectedPortNum, expectedHostGroupNameFour));
+ hdfsSiteProperties.put("dfs.namenode.servicerpc-address." +
expectedNameServiceTwo + "." + expectedNodeThree,
createExportedAddress(expectedPortNum, expectedHostGroupNameThree));
+ hdfsSiteProperties.put("dfs.namenode.servicerpc-address." +
expectedNameServiceTwo + "." + expectedNodeFour,
createExportedAddress(expectedPortNum, expectedHostGroupNameFour));
+
+
+ // add properties that require the SECONDARY_NAMENODE, which
+ // is not included in this test
+ hdfsSiteProperties.put("dfs.secondary.http.address", "localhost:8080");
+ hdfsSiteProperties.put("dfs.namenode.secondary.http-address",
"localhost:8080");
+
+
+ // add properties that are used in non-HA HDFS NameNode settings
+ // to verify that these are eventually removed by the filter
+ hdfsSiteProperties.put("dfs.namenode.http-address", "localhost:8080");
+ hdfsSiteProperties.put("dfs.namenode.https-address", "localhost:8081");
+ hdfsSiteProperties.put("dfs.namenode.rpc-address", "localhost:8082");
+
+ // configure the defaultFS to use the nameservice URL
+ coreSiteProperties.put("fs.defaultFS", "hdfs://" + expectedNameService);
+
+ // configure the hbase rootdir to use the nameservice URL
+ hbaseSiteProperties.put("hbase.rootdir", "hdfs://" + expectedNameService +
"/hbase/test/root/dir");
+
+ // configure the hbase rootdir to use the nameservice URL
+ accumuloSiteProperties.put("instance.volumes", "hdfs://" +
expectedNameService + "/accumulo/test/instance/volumes");
+
+ rangerHDFSPluginProperties.put("hadoop.rpc.protection","authentication");
+ rangerHDFSPluginProperties.put("REPOSITORY_CONFIG_USERNAME","hadoop");
+ rangerHDFSPluginProperties.put("REPOSITORY_CONFIG_PASSWORD","hadoop");
+ rangerHDFSPluginProperties.put("ranger-hdfs-plugin-enabled","Yes");
+
+
rangerHDFSSecurityProperties.put("ranger.plugin.hdfs.service.name","{{repo_name}}");
+
+ Configuration clusterConfig = new Configuration(properties, emptyMap());
+
+ Collection<String> hgComponents = new HashSet<>();
+ hgComponents.add("NAMENODE");
+ hgComponents.add("RANGER_ADMIN");
+ hgComponents.add("RANGER_USERSYNC");
+ hgComponents.add("RANGER_TAGSYNC");
+ TestHostGroup group1 = new TestHostGroup(expectedHostGroupName,
hgComponents, Collections.singleton(expectedHostName));
+
+ Collection<String> hgComponents2 = new HashSet<>();
+ hgComponents2.add("NAMENODE");
+ hgComponents2.add("ATLAS_SERVER");
+ hgComponents2.add("ATLAS_CLIENT");
+ TestHostGroup group2 = new TestHostGroup(expectedHostGroupNameTwo,
hgComponents2, Collections.singleton(expectedHostNameTwo));
+
+ // add third and fourth hostgroup with NAMENODE, to simulate HDFS NameNode
Federation
+ TestHostGroup group3 = new TestHostGroup(expectedHostGroupNameThree,
Collections.singleton("NAMENODE"),
Collections.singleton(expectedHostNameThree));
+ TestHostGroup group4 = new TestHostGroup(expectedHostGroupNameFour,
Collections.singleton("NAMENODE"), Collections.singleton(expectedHostNameFour));
+
+ Collection<TestHostGroup> hostGroups = new ArrayList<>();
+ hostGroups.add(group1);
+ hostGroups.add(group2);
+ hostGroups.add(group3);
+ hostGroups.add(group4);
+
+ expect(stack.getCardinality("NAMENODE")).andReturn(new
Cardinality("1-2")).anyTimes();
+ expect(stack.getCardinality("SECONDARY_NAMENODE")).andReturn(new
Cardinality("1")).anyTimes();
+
+ ClusterTopology topology = createClusterTopology(bp, clusterConfig,
hostGroups);
+ BlueprintConfigurationProcessor updater = new
BlueprintConfigurationProcessor(topology);
+
+ Set<String> updatedConfigTypes =
+ updater.doUpdateForClusterCreate();
+
+ // verify that correct configuration types were listed as updated in the
returned set
+ assertEquals(ImmutableSet.of("cluster-env", "hdfs-site",
"ranger-tagsync-site"), updatedConfigTypes);
+
+
+ Map<String, String> updatedRangerTagsyncSiteConfigurations =
clusterConfig.getProperties().get("ranger-tagsync-site");
+ assertTrue("Expected property
ranger.tagsync.atlas.hdfs.instance.clusterName.nameservice.mynameservice.ranger.service
not found.",
updatedRangerTagsyncSiteConfigurations.containsKey("ranger.tagsync.atlas.hdfs.instance.clusterName.nameservice.mynameservice.ranger.service"));
+ assertTrue("Expected property
ranger.tagsync.atlas.hdfs.instance.clusterName.nameservice.mynameservicetwo.ranger.service
not found.",
updatedRangerTagsyncSiteConfigurations.containsKey("ranger.tagsync.atlas.hdfs.instance.clusterName.nameservice.mynameservicetwo.ranger.service"));
+ assertTrue("Expected property
ranger.tagsync.atlas.hdfs.instance.clusterName.ranger.service not found.",
updatedRangerTagsyncSiteConfigurations.containsKey("ranger.tagsync.atlas.hdfs.instance.clusterName.ranger.service"));
+ assertEquals("Expected name service clusterName_hadoop_mynameservice not
found",
"clusterName_hadoop_mynameservice",updatedRangerTagsyncSiteConfigurations.get("ranger.tagsync.atlas.hdfs.instance.clusterName.nameservice.mynameservice.ranger.service"));
+ assertEquals("Expected name service clusterName_hadoop_mynameservicetwo
not found",
"clusterName_hadoop_mynameservicetwo",updatedRangerTagsyncSiteConfigurations.get("ranger.tagsync.atlas.hdfs.instance.clusterName.nameservice.mynameservicetwo.ranger.service"));
+ assertEquals("Expected name service clusterName_hadoop_mynameservicetwo
not found",
"clusterName_hadoop_mynameservice",updatedRangerTagsyncSiteConfigurations.get("ranger.tagsync.atlas.hdfs.instance.clusterName.ranger.service"));
+ }
+
+ @Test
+ public void
testDoUpdateForClusterWithNameNodeFederationEnabledTagsyncEnabledCustomRepoName()
throws Exception {
+ final String expectedNameService = "mynameservice";
+ final String expectedNameServiceTwo = "mynameservicetwo";
+ final String expectedHostName = "c6401.apache.ambari.org";
+ final String expectedHostNameTwo = "c6402.apache.ambari.org";
+ final String expectedHostNameThree = "c6403.apache.ambari.org";
+ final String expectedHostNameFour = "c6404.apache.ambari.org";
+ final String expectedPortNum = "808080";
+ final String expectedNodeOne = "nn1";
+ final String expectedNodeTwo = "nn2";
+ final String expectedNodeThree = "nn3";
+ final String expectedNodeFour = "nn4";
+ final String expectedHostGroupName = "host_group_1";
+ final String expectedHostGroupNameTwo = "host_group_2";
+ final String expectedHostGroupNameThree = "host-group-3";
+ final String expectedHostGroupNameFour = "host-group-4";
+
+ Map<String, Map<String, String>> properties = new HashMap<>();
+
+ Map<String, String> hdfsSiteProperties = new HashMap<>();
+ Map<String, String> hbaseSiteProperties = new HashMap<>();
+ Map<String, String> hadoopEnvProperties = new HashMap<>();
+ Map<String, String> coreSiteProperties = new HashMap<>();
+ Map<String, String> accumuloSiteProperties = new HashMap<>();
+ Map<String, String> rangerHDFSPluginProperties = new HashMap<>();
+ Map<String, String> rangerHDFSSecurityProperties = new HashMap<>();
+
+ properties.put("hdfs-site", hdfsSiteProperties);
+ properties.put("hadoop-env", hadoopEnvProperties);
+ properties.put("core-site", coreSiteProperties);
+ properties.put("hbase-site", hbaseSiteProperties);
+ properties.put("accumulo-site", accumuloSiteProperties);
+ properties.put("ranger-hdfs-plugin-properties",
rangerHDFSPluginProperties);
+ properties.put("ranger-hdfs-security", rangerHDFSSecurityProperties);
+
+ // setup multiple nameservices, to indicate NameNode Federation will be
used
+ hdfsSiteProperties.put("dfs.nameservices", expectedNameService + "," +
expectedNameServiceTwo);
+ hdfsSiteProperties.put("dfs.ha.namenodes.mynameservice", expectedNodeOne +
", " + expectedNodeTwo);
+ hdfsSiteProperties.put("dfs.ha.namenodes." + expectedNameServiceTwo,
expectedNodeThree + "," + expectedNodeFour);
+
+ //setup nameservice-specific properties
+ hdfsSiteProperties.put("dfs.namenode.shared.edits.dir" + "." +
expectedNameService,
+ "qjournal://" + createExportedAddress(expectedPortNum,
expectedHostGroupName) + ";" + createExportedAddress(expectedPortNum,
expectedHostGroupNameTwo) + "/ns1");
+ hdfsSiteProperties.put("dfs.namenode.shared.edits.dir" + "." +
expectedNameServiceTwo,
+ "qjournal://" + createExportedAddress(expectedPortNum,
expectedHostGroupName) + ";" + createExportedAddress(expectedPortNum,
expectedHostGroupNameTwo) + "/ns2");
+
+
+ // setup properties that include exported host group information
+ hdfsSiteProperties.put("dfs.namenode.https-address." + expectedNameService
+ "." + expectedNodeOne, createExportedAddress(expectedPortNum,
expectedHostGroupName));
+ hdfsSiteProperties.put("dfs.namenode.https-address." + expectedNameService
+ "." + expectedNodeTwo, createExportedAddress(expectedPortNum,
expectedHostGroupNameTwo));
+ hdfsSiteProperties.put("dfs.namenode.http-address." + expectedNameService
+ "." + expectedNodeOne, createExportedAddress(expectedPortNum,
expectedHostGroupName));
+ hdfsSiteProperties.put("dfs.namenode.http-address." + expectedNameService
+ "." + expectedNodeTwo, createExportedAddress(expectedPortNum,
expectedHostGroupNameTwo));
+ hdfsSiteProperties.put("dfs.namenode.rpc-address." + expectedNameService +
"." + expectedNodeOne, createExportedAddress(expectedPortNum,
expectedHostGroupName));
+ hdfsSiteProperties.put("dfs.namenode.rpc-address." + expectedNameService +
"." + expectedNodeTwo, createExportedAddress(expectedPortNum,
expectedHostGroupNameTwo));
+ hdfsSiteProperties.put("dfs.namenode.servicerpc-address." +
expectedNameService + "." + expectedNodeOne,
createExportedAddress(expectedPortNum, expectedHostGroupName));
+ hdfsSiteProperties.put("dfs.namenode.servicerpc-address." +
expectedNameService + "." + expectedNodeTwo,
createExportedAddress(expectedPortNum, expectedHostGroupNameTwo));
+
+ hdfsSiteProperties.put("dfs.namenode.https-address." +
expectedNameServiceTwo + "." + expectedNodeThree,
createExportedAddress(expectedPortNum, expectedHostGroupNameThree));
+ hdfsSiteProperties.put("dfs.namenode.https-address." +
expectedNameServiceTwo + "." + expectedNodeFour,
createExportedAddress(expectedPortNum, expectedHostGroupNameFour));
+ hdfsSiteProperties.put("dfs.namenode.http-address." +
expectedNameServiceTwo + "." + expectedNodeThree,
createExportedAddress(expectedPortNum, expectedHostGroupNameThree));
+ hdfsSiteProperties.put("dfs.namenode.http-address." +
expectedNameServiceTwo + "." + expectedNodeFour,
createExportedAddress(expectedPortNum, expectedHostGroupNameFour));
+ hdfsSiteProperties.put("dfs.namenode.rpc-address." +
expectedNameServiceTwo + "." + expectedNodeThree,
createExportedAddress(expectedPortNum, expectedHostGroupNameThree));
+ hdfsSiteProperties.put("dfs.namenode.rpc-address." +
expectedNameServiceTwo + "." + expectedNodeFour,
createExportedAddress(expectedPortNum, expectedHostGroupNameFour));
+ hdfsSiteProperties.put("dfs.namenode.servicerpc-address." +
expectedNameServiceTwo + "." + expectedNodeThree,
createExportedAddress(expectedPortNum, expectedHostGroupNameThree));
+ hdfsSiteProperties.put("dfs.namenode.servicerpc-address." +
expectedNameServiceTwo + "." + expectedNodeFour,
createExportedAddress(expectedPortNum, expectedHostGroupNameFour));
+
+
+ // add properties that require the SECONDARY_NAMENODE, which
+ // is not included in this test
+ hdfsSiteProperties.put("dfs.secondary.http.address", "localhost:8080");
+ hdfsSiteProperties.put("dfs.namenode.secondary.http-address",
"localhost:8080");
+
+
+ // add properties that are used in non-HA HDFS NameNode settings
+ // to verify that these are eventually removed by the filter
+ hdfsSiteProperties.put("dfs.namenode.http-address", "localhost:8080");
+ hdfsSiteProperties.put("dfs.namenode.https-address", "localhost:8081");
+ hdfsSiteProperties.put("dfs.namenode.rpc-address", "localhost:8082");
+
+ // configure the defaultFS to use the nameservice URL
+ coreSiteProperties.put("fs.defaultFS", "hdfs://" + expectedNameService);
+
+ // configure the hbase rootdir to use the nameservice URL
+ hbaseSiteProperties.put("hbase.rootdir", "hdfs://" + expectedNameService +
"/hbase/test/root/dir");
+
+ // configure the hbase rootdir to use the nameservice URL
+ accumuloSiteProperties.put("instance.volumes", "hdfs://" +
expectedNameService + "/accumulo/test/instance/volumes");
+
+ rangerHDFSPluginProperties.put("hadoop.rpc.protection","authentication");
+ rangerHDFSPluginProperties.put("REPOSITORY_CONFIG_USERNAME","hadoop");
+ rangerHDFSPluginProperties.put("REPOSITORY_CONFIG_PASSWORD","hadoop");
+ rangerHDFSPluginProperties.put("ranger-hdfs-plugin-enabled","Yes");
+
+
rangerHDFSSecurityProperties.put("ranger.plugin.hdfs.service.name","hdfs_service");
+
+ Configuration clusterConfig = new Configuration(properties, emptyMap());
+
+ Collection<String> hgComponents = new HashSet<>();
+ hgComponents.add("NAMENODE");
+ hgComponents.add("RANGER_ADMIN");
+ hgComponents.add("RANGER_USERSYNC");
+ hgComponents.add("RANGER_TAGSYNC");
+ TestHostGroup group1 = new TestHostGroup(expectedHostGroupName,
hgComponents, Collections.singleton(expectedHostName));
+
+ Collection<String> hgComponents2 = new HashSet<>();
+ hgComponents2.add("NAMENODE");
+ hgComponents2.add("ATLAS_SERVER");
+ hgComponents2.add("ATLAS_CLIENT");
+ TestHostGroup group2 = new TestHostGroup(expectedHostGroupNameTwo,
hgComponents2, Collections.singleton(expectedHostNameTwo));
+
+ // add third and fourth hostgroup with NAMENODE, to simulate HDFS NameNode
Federation
+ TestHostGroup group3 = new TestHostGroup(expectedHostGroupNameThree,
Collections.singleton("NAMENODE"),
Collections.singleton(expectedHostNameThree));
+ TestHostGroup group4 = new TestHostGroup(expectedHostGroupNameFour,
Collections.singleton("NAMENODE"), Collections.singleton(expectedHostNameFour));
+
+ Collection<TestHostGroup> hostGroups = new ArrayList<>();
+ hostGroups.add(group1);
+ hostGroups.add(group2);
+ hostGroups.add(group3);
+ hostGroups.add(group4);
+
+ expect(stack.getCardinality("NAMENODE")).andReturn(new
Cardinality("1-2")).anyTimes();
+ expect(stack.getCardinality("SECONDARY_NAMENODE")).andReturn(new
Cardinality("1")).anyTimes();
+
+ ClusterTopology topology = createClusterTopology(bp, clusterConfig,
hostGroups);
+ BlueprintConfigurationProcessor updater = new
BlueprintConfigurationProcessor(topology);
+
+ Set<String> updatedConfigTypes =
+ updater.doUpdateForClusterCreate();
+
+ // verify that correct configuration types were listed as updated in the
returned set
+ assertEquals(ImmutableSet.of("cluster-env", "hdfs-site",
"ranger-tagsync-site"), updatedConfigTypes);
+
+
+ Map<String, String> updatedRangerTagsyncSiteConfigurations =
clusterConfig.getProperties().get("ranger-tagsync-site");
+ assertTrue("Expected property
ranger.tagsync.atlas.hdfs.instance.clusterName.nameservice.mynameservice.ranger.service
not found.",
updatedRangerTagsyncSiteConfigurations.containsKey("ranger.tagsync.atlas.hdfs.instance.clusterName.nameservice.mynameservice.ranger.service"));
+ assertTrue("Expected property
ranger.tagsync.atlas.hdfs.instance.clusterName.nameservice.mynameservicetwo.ranger.service
not found.",
updatedRangerTagsyncSiteConfigurations.containsKey("ranger.tagsync.atlas.hdfs.instance.clusterName.nameservice.mynameservicetwo.ranger.service"));
+ assertTrue("Expected property
ranger.tagsync.atlas.hdfs.instance.clusterName.ranger.service not found.",
updatedRangerTagsyncSiteConfigurations.containsKey("ranger.tagsync.atlas.hdfs.instance.clusterName.ranger.service"));
+ assertEquals("Expected name service hdfs_service_mynameservice not found",
"hdfs_service_mynameservice",updatedRangerTagsyncSiteConfigurations.get("ranger.tagsync.atlas.hdfs.instance.clusterName.nameservice.mynameservice.ranger.service"));
+ assertEquals("Expected name service hdfs_service_mynameservicetwo not
found",
"hdfs_service_mynameservicetwo",updatedRangerTagsyncSiteConfigurations.get("ranger.tagsync.atlas.hdfs.instance.clusterName.nameservice.mynameservicetwo.ranger.service"));
+ assertEquals("Expected name service hdfs_service_mynameservicetwo not
found",
"hdfs_service_mynameservice",updatedRangerTagsyncSiteConfigurations.get("ranger.tagsync.atlas.hdfs.instance.clusterName.ranger.service"));
+ }
+
+ @Test
+ public void
testDoUpdateForClusterWithNameNodeFederationEnabledTagsyncEnabledPluginDisabled()
throws Exception {
+ final String expectedNameService = "mynameservice";
+ final String expectedNameServiceTwo = "mynameservicetwo";
+ final String expectedHostName = "c6401.apache.ambari.org";
+ final String expectedHostNameTwo = "c6402.apache.ambari.org";
+ final String expectedHostNameThree = "c6403.apache.ambari.org";
+ final String expectedHostNameFour = "c6404.apache.ambari.org";
+ final String expectedPortNum = "808080";
+ final String expectedNodeOne = "nn1";
+ final String expectedNodeTwo = "nn2";
+ final String expectedNodeThree = "nn3";
+ final String expectedNodeFour = "nn4";
+ final String expectedHostGroupName = "host_group_1";
+ final String expectedHostGroupNameTwo = "host_group_2";
+ final String expectedHostGroupNameThree = "host-group-3";
+ final String expectedHostGroupNameFour = "host-group-4";
+
+ Map<String, Map<String, String>> properties = new HashMap<>();
+
+ Map<String, String> hdfsSiteProperties = new HashMap<>();
+ Map<String, String> hbaseSiteProperties = new HashMap<>();
+ Map<String, String> hadoopEnvProperties = new HashMap<>();
+ Map<String, String> coreSiteProperties = new HashMap<>();
+ Map<String, String> accumuloSiteProperties = new HashMap<>();
+ Map<String, String> rangerHDFSPluginProperties = new HashMap<>();
+ Map<String, String> rangerHDFSSecurityProperties = new HashMap<>();
+
+ properties.put("hdfs-site", hdfsSiteProperties);
+ properties.put("hadoop-env", hadoopEnvProperties);
+ properties.put("core-site", coreSiteProperties);
+ properties.put("hbase-site", hbaseSiteProperties);
+ properties.put("accumulo-site", accumuloSiteProperties);
+ properties.put("ranger-hdfs-plugin-properties",
rangerHDFSPluginProperties);
+ properties.put("ranger-hdfs-security", rangerHDFSSecurityProperties);
+
+ // setup multiple nameservices, to indicate NameNode Federation will be
used
+ hdfsSiteProperties.put("dfs.nameservices", expectedNameService + "," +
expectedNameServiceTwo);
+ hdfsSiteProperties.put("dfs.ha.namenodes.mynameservice", expectedNodeOne +
", " + expectedNodeTwo);
+ hdfsSiteProperties.put("dfs.ha.namenodes." + expectedNameServiceTwo,
expectedNodeThree + "," + expectedNodeFour);
+
+ //setup nameservice-specific properties
+ hdfsSiteProperties.put("dfs.namenode.shared.edits.dir" + "." +
expectedNameService,
+ "qjournal://" + createExportedAddress(expectedPortNum,
expectedHostGroupName) + ";" + createExportedAddress(expectedPortNum,
expectedHostGroupNameTwo) + "/ns1");
+ hdfsSiteProperties.put("dfs.namenode.shared.edits.dir" + "." +
expectedNameServiceTwo,
+ "qjournal://" + createExportedAddress(expectedPortNum,
expectedHostGroupName) + ";" + createExportedAddress(expectedPortNum,
expectedHostGroupNameTwo) + "/ns2");
+
+
+ // setup properties that include exported host group information
+ hdfsSiteProperties.put("dfs.namenode.https-address." + expectedNameService
+ "." + expectedNodeOne, createExportedAddress(expectedPortNum,
expectedHostGroupName));
+ hdfsSiteProperties.put("dfs.namenode.https-address." + expectedNameService
+ "." + expectedNodeTwo, createExportedAddress(expectedPortNum,
expectedHostGroupNameTwo));
+ hdfsSiteProperties.put("dfs.namenode.http-address." + expectedNameService
+ "." + expectedNodeOne, createExportedAddress(expectedPortNum,
expectedHostGroupName));
+ hdfsSiteProperties.put("dfs.namenode.http-address." + expectedNameService
+ "." + expectedNodeTwo, createExportedAddress(expectedPortNum,
expectedHostGroupNameTwo));
+ hdfsSiteProperties.put("dfs.namenode.rpc-address." + expectedNameService +
"." + expectedNodeOne, createExportedAddress(expectedPortNum,
expectedHostGroupName));
+ hdfsSiteProperties.put("dfs.namenode.rpc-address." + expectedNameService +
"." + expectedNodeTwo, createExportedAddress(expectedPortNum,
expectedHostGroupNameTwo));
+ hdfsSiteProperties.put("dfs.namenode.servicerpc-address." +
expectedNameService + "." + expectedNodeOne,
createExportedAddress(expectedPortNum, expectedHostGroupName));
+ hdfsSiteProperties.put("dfs.namenode.servicerpc-address." +
expectedNameService + "." + expectedNodeTwo,
createExportedAddress(expectedPortNum, expectedHostGroupNameTwo));
+
+ hdfsSiteProperties.put("dfs.namenode.https-address." +
expectedNameServiceTwo + "." + expectedNodeThree,
createExportedAddress(expectedPortNum, expectedHostGroupNameThree));
+ hdfsSiteProperties.put("dfs.namenode.https-address." +
expectedNameServiceTwo + "." + expectedNodeFour,
createExportedAddress(expectedPortNum, expectedHostGroupNameFour));
+ hdfsSiteProperties.put("dfs.namenode.http-address." +
expectedNameServiceTwo + "." + expectedNodeThree,
createExportedAddress(expectedPortNum, expectedHostGroupNameThree));
+ hdfsSiteProperties.put("dfs.namenode.http-address." +
expectedNameServiceTwo + "." + expectedNodeFour,
createExportedAddress(expectedPortNum, expectedHostGroupNameFour));
+ hdfsSiteProperties.put("dfs.namenode.rpc-address." +
expectedNameServiceTwo + "." + expectedNodeThree,
createExportedAddress(expectedPortNum, expectedHostGroupNameThree));
+ hdfsSiteProperties.put("dfs.namenode.rpc-address." +
expectedNameServiceTwo + "." + expectedNodeFour,
createExportedAddress(expectedPortNum, expectedHostGroupNameFour));
+ hdfsSiteProperties.put("dfs.namenode.servicerpc-address." +
expectedNameServiceTwo + "." + expectedNodeThree,
createExportedAddress(expectedPortNum, expectedHostGroupNameThree));
+ hdfsSiteProperties.put("dfs.namenode.servicerpc-address." +
expectedNameServiceTwo + "." + expectedNodeFour,
createExportedAddress(expectedPortNum, expectedHostGroupNameFour));
+
+
+ // add properties that require the SECONDARY_NAMENODE, which
+ // is not included in this test
+ hdfsSiteProperties.put("dfs.secondary.http.address", "localhost:8080");
+ hdfsSiteProperties.put("dfs.namenode.secondary.http-address",
"localhost:8080");
+
+
+ // add properties that are used in non-HA HDFS NameNode settings
+ // to verify that these are eventually removed by the filter
+ hdfsSiteProperties.put("dfs.namenode.http-address", "localhost:8080");
+ hdfsSiteProperties.put("dfs.namenode.https-address", "localhost:8081");
+ hdfsSiteProperties.put("dfs.namenode.rpc-address", "localhost:8082");
+
+ // configure the defaultFS to use the nameservice URL
+ coreSiteProperties.put("fs.defaultFS", "hdfs://" + expectedNameService);
+
+ // configure the hbase rootdir to use the nameservice URL
+ hbaseSiteProperties.put("hbase.rootdir", "hdfs://" + expectedNameService +
"/hbase/test/root/dir");
+
+ // configure the hbase rootdir to use the nameservice URL
+ accumuloSiteProperties.put("instance.volumes", "hdfs://" +
expectedNameService + "/accumulo/test/instance/volumes");
+
+ rangerHDFSPluginProperties.put("hadoop.rpc.protection","authentication");
+ rangerHDFSPluginProperties.put("REPOSITORY_CONFIG_USERNAME","hadoop");
+ rangerHDFSPluginProperties.put("REPOSITORY_CONFIG_PASSWORD","hadoop");
+ rangerHDFSPluginProperties.put("ranger-hdfs-plugin-enabled","No");
+
+
rangerHDFSSecurityProperties.put("ranger.plugin.hdfs.service.name","{{repo_name}}");
+
+ Configuration clusterConfig = new Configuration(properties, emptyMap());
+
+ Collection<String> hgComponents = new HashSet<>();
+ hgComponents.add("NAMENODE");
+ hgComponents.add("RANGER_ADMIN");
+ hgComponents.add("RANGER_USERSYNC");
+ hgComponents.add("RANGER_TAGSYNC");
+ TestHostGroup group1 = new TestHostGroup(expectedHostGroupName,
hgComponents, Collections.singleton(expectedHostName));
+
+ Collection<String> hgComponents2 = new HashSet<>();
+ hgComponents2.add("NAMENODE");
+ hgComponents2.add("ATLAS_SERVER");
+ hgComponents2.add("ATLAS_CLIENT");
+ TestHostGroup group2 = new TestHostGroup(expectedHostGroupNameTwo,
hgComponents2, Collections.singleton(expectedHostNameTwo));
+
+ // add third and fourth hostgroup with NAMENODE, to simulate HDFS NameNode
Federation
+ TestHostGroup group3 = new TestHostGroup(expectedHostGroupNameThree,
Collections.singleton("NAMENODE"),
Collections.singleton(expectedHostNameThree));
+ TestHostGroup group4 = new TestHostGroup(expectedHostGroupNameFour,
Collections.singleton("NAMENODE"), Collections.singleton(expectedHostNameFour));
+
+ Collection<TestHostGroup> hostGroups = new ArrayList<>();
+ hostGroups.add(group1);
+ hostGroups.add(group2);
+ hostGroups.add(group3);
+ hostGroups.add(group4);
+
+ expect(stack.getCardinality("NAMENODE")).andReturn(new
Cardinality("1-2")).anyTimes();
+ expect(stack.getCardinality("SECONDARY_NAMENODE")).andReturn(new
Cardinality("1")).anyTimes();
+
+ ClusterTopology topology = createClusterTopology(bp, clusterConfig,
hostGroups);
+ BlueprintConfigurationProcessor updater = new
BlueprintConfigurationProcessor(topology);
+
+ Set<String> updatedConfigTypes = updater.doUpdateForClusterCreate();
+
+ // verify that correct configuration types were listed as updated in the
returned set
+ assertEquals(ImmutableSet.of("cluster-env", "hdfs-site"),
updatedConfigTypes);
+
+
+ Map<String, String> updatedRangerTagsyncSiteConfigurations =
clusterConfig.getProperties().get("ranger-tagsync-site");
+ assertNull(updatedRangerTagsyncSiteConfigurations);
+
+
+ }
+
+ @Test
+ public void
testDoUpdateForClusterWithNameNodeFederationEnabledTagsyncEnabledPluginEnabledNoAtlas()
throws Exception {
+ final String expectedNameService = "mynameservice";
+ final String expectedNameServiceTwo = "mynameservicetwo";
+ final String expectedHostName = "c6401.apache.ambari.org";
+ final String expectedHostNameTwo = "c6402.apache.ambari.org";
+ final String expectedHostNameThree = "c6403.apache.ambari.org";
+ final String expectedHostNameFour = "c6404.apache.ambari.org";
+ final String expectedPortNum = "808080";
+ final String expectedNodeOne = "nn1";
+ final String expectedNodeTwo = "nn2";
+ final String expectedNodeThree = "nn3";
+ final String expectedNodeFour = "nn4";
+ final String expectedHostGroupName = "host_group_1";
+ final String expectedHostGroupNameTwo = "host_group_2";
+ final String expectedHostGroupNameThree = "host-group-3";
+ final String expectedHostGroupNameFour = "host-group-4";
+
+ Map<String, Map<String, String>> properties = new HashMap<>();
+
+ Map<String, String> hdfsSiteProperties = new HashMap<>();
+ Map<String, String> hbaseSiteProperties = new HashMap<>();
+ Map<String, String> hadoopEnvProperties = new HashMap<>();
+ Map<String, String> coreSiteProperties = new HashMap<>();
+ Map<String, String> accumuloSiteProperties = new HashMap<>();
+ Map<String, String> rangerHDFSPluginProperties = new HashMap<>();
+ Map<String, String> rangerHDFSSecurityProperties = new HashMap<>();
+
+ properties.put("hdfs-site", hdfsSiteProperties);
+ properties.put("hadoop-env", hadoopEnvProperties);
+ properties.put("core-site", coreSiteProperties);
+ properties.put("hbase-site", hbaseSiteProperties);
+ properties.put("accumulo-site", accumuloSiteProperties);
+ properties.put("ranger-hdfs-plugin-properties",
rangerHDFSPluginProperties);
+ properties.put("ranger-hdfs-security", rangerHDFSSecurityProperties);
+
+ // setup multiple nameservices, to indicate NameNode Federation will be
used
+ hdfsSiteProperties.put("dfs.nameservices", expectedNameService + "," +
expectedNameServiceTwo);
+ hdfsSiteProperties.put("dfs.ha.namenodes.mynameservice", expectedNodeOne +
", " + expectedNodeTwo);
+ hdfsSiteProperties.put("dfs.ha.namenodes." + expectedNameServiceTwo,
expectedNodeThree + "," + expectedNodeFour);
+
+ //setup nameservice-specific properties
+ hdfsSiteProperties.put("dfs.namenode.shared.edits.dir" + "." +
expectedNameService,
+ "qjournal://" + createExportedAddress(expectedPortNum,
expectedHostGroupName) + ";" + createExportedAddress(expectedPortNum,
expectedHostGroupNameTwo) + "/ns1");
+ hdfsSiteProperties.put("dfs.namenode.shared.edits.dir" + "." +
expectedNameServiceTwo,
+ "qjournal://" + createExportedAddress(expectedPortNum,
expectedHostGroupName) + ";" + createExportedAddress(expectedPortNum,
expectedHostGroupNameTwo) + "/ns2");
+
+
+ // setup properties that include exported host group information
+ hdfsSiteProperties.put("dfs.namenode.https-address." + expectedNameService
+ "." + expectedNodeOne, createExportedAddress(expectedPortNum,
expectedHostGroupName));
+ hdfsSiteProperties.put("dfs.namenode.https-address." + expectedNameService
+ "." + expectedNodeTwo, createExportedAddress(expectedPortNum,
expectedHostGroupNameTwo));
+ hdfsSiteProperties.put("dfs.namenode.http-address." + expectedNameService
+ "." + expectedNodeOne, createExportedAddress(expectedPortNum,
expectedHostGroupName));
+ hdfsSiteProperties.put("dfs.namenode.http-address." + expectedNameService
+ "." + expectedNodeTwo, createExportedAddress(expectedPortNum,
expectedHostGroupNameTwo));
+ hdfsSiteProperties.put("dfs.namenode.rpc-address." + expectedNameService +
"." + expectedNodeOne, createExportedAddress(expectedPortNum,
expectedHostGroupName));
+ hdfsSiteProperties.put("dfs.namenode.rpc-address." + expectedNameService +
"." + expectedNodeTwo, createExportedAddress(expectedPortNum,
expectedHostGroupNameTwo));
+ hdfsSiteProperties.put("dfs.namenode.servicerpc-address." +
expectedNameService + "." + expectedNodeOne,
createExportedAddress(expectedPortNum, expectedHostGroupName));
+ hdfsSiteProperties.put("dfs.namenode.servicerpc-address." +
expectedNameService + "." + expectedNodeTwo,
createExportedAddress(expectedPortNum, expectedHostGroupNameTwo));
+
+ hdfsSiteProperties.put("dfs.namenode.https-address." +
expectedNameServiceTwo + "." + expectedNodeThree,
createExportedAddress(expectedPortNum, expectedHostGroupNameThree));
+ hdfsSiteProperties.put("dfs.namenode.https-address." +
expectedNameServiceTwo + "." + expectedNodeFour,
createExportedAddress(expectedPortNum, expectedHostGroupNameFour));
+ hdfsSiteProperties.put("dfs.namenode.http-address." +
expectedNameServiceTwo + "." + expectedNodeThree,
createExportedAddress(expectedPortNum, expectedHostGroupNameThree));
+ hdfsSiteProperties.put("dfs.namenode.http-address." +
expectedNameServiceTwo + "." + expectedNodeFour,
createExportedAddress(expectedPortNum, expectedHostGroupNameFour));
+ hdfsSiteProperties.put("dfs.namenode.rpc-address." +
expectedNameServiceTwo + "." + expectedNodeThree,
createExportedAddress(expectedPortNum, expectedHostGroupNameThree));
+ hdfsSiteProperties.put("dfs.namenode.rpc-address." +
expectedNameServiceTwo + "." + expectedNodeFour,
createExportedAddress(expectedPortNum, expectedHostGroupNameFour));
+ hdfsSiteProperties.put("dfs.namenode.servicerpc-address." +
expectedNameServiceTwo + "." + expectedNodeThree,
createExportedAddress(expectedPortNum, expectedHostGroupNameThree));
+ hdfsSiteProperties.put("dfs.namenode.servicerpc-address." +
expectedNameServiceTwo + "." + expectedNodeFour,
createExportedAddress(expectedPortNum, expectedHostGroupNameFour));
+
+
+ // add properties that require the SECONDARY_NAMENODE, which
+ // is not included in this test
+ hdfsSiteProperties.put("dfs.secondary.http.address", "localhost:8080");
+ hdfsSiteProperties.put("dfs.namenode.secondary.http-address",
"localhost:8080");
+
+
+ // add properties that are used in non-HA HDFS NameNode settings
+ // to verify that these are eventually removed by the filter
+ hdfsSiteProperties.put("dfs.namenode.http-address", "localhost:8080");
+ hdfsSiteProperties.put("dfs.namenode.https-address", "localhost:8081");
+ hdfsSiteProperties.put("dfs.namenode.rpc-address", "localhost:8082");
+
+ // configure the defaultFS to use the nameservice URL
+ coreSiteProperties.put("fs.defaultFS", "hdfs://" + expectedNameService);
+
+ // configure the hbase rootdir to use the nameservice URL
+ hbaseSiteProperties.put("hbase.rootdir", "hdfs://" + expectedNameService +
"/hbase/test/root/dir");
+
+ // configure the hbase rootdir to use the nameservice URL
+ accumuloSiteProperties.put("instance.volumes", "hdfs://" +
expectedNameService + "/accumulo/test/instance/volumes");
+
+ rangerHDFSPluginProperties.put("hadoop.rpc.protection","authentication");
+ rangerHDFSPluginProperties.put("REPOSITORY_CONFIG_USERNAME","hadoop");
+ rangerHDFSPluginProperties.put("REPOSITORY_CONFIG_PASSWORD","hadoop");
+ rangerHDFSPluginProperties.put("ranger-hdfs-plugin-enabled","Yes");
+
+
rangerHDFSSecurityProperties.put("ranger.plugin.hdfs.service.name","{{repo_name}}");
+
+ Configuration clusterConfig = new Configuration(properties, emptyMap());
+
+ Collection<String> hgComponents = new HashSet<>();
+ hgComponents.add("NAMENODE");
+ hgComponents.add("RANGER_ADMIN");
+ hgComponents.add("RANGER_USERSYNC");
+ hgComponents.add("RANGER_TAGSYNC");
+ TestHostGroup group1 = new TestHostGroup(expectedHostGroupName,
hgComponents, Collections.singleton(expectedHostName));
+
+ Collection<String> hgComponents2 = new HashSet<>();
+ hgComponents2.add("NAMENODE");
+ TestHostGroup group2 = new TestHostGroup(expectedHostGroupNameTwo,
hgComponents2, Collections.singleton(expectedHostNameTwo));
+
+ // add third and fourth hostgroup with NAMENODE, to simulate HDFS NameNode
Federation
+ TestHostGroup group3 = new TestHostGroup(expectedHostGroupNameThree,
Collections.singleton("NAMENODE"),
Collections.singleton(expectedHostNameThree));
+ TestHostGroup group4 = new TestHostGroup(expectedHostGroupNameFour,
Collections.singleton("NAMENODE"), Collections.singleton(expectedHostNameFour));
+
+ Collection<TestHostGroup> hostGroups = new ArrayList<>();
+ hostGroups.add(group1);
+ hostGroups.add(group2);
+ hostGroups.add(group3);
+ hostGroups.add(group4);
+
+ expect(stack.getCardinality("NAMENODE")).andReturn(new
Cardinality("1-2")).anyTimes();
+ expect(stack.getCardinality("SECONDARY_NAMENODE")).andReturn(new
Cardinality("1")).anyTimes();
+
+ ClusterTopology topology = createClusterTopology(bp, clusterConfig,
hostGroups);
+ BlueprintConfigurationProcessor updater = new
BlueprintConfigurationProcessor(topology);
+
+ Set<String> updatedConfigTypes = updater.doUpdateForClusterCreate();
+
+ // verify that correct configuration types were listed as updated in the
returned set
+ assertEquals(ImmutableSet.of("cluster-env", "hdfs-site"),
updatedConfigTypes);
+
+
+ Map<String, String> updatedRangerTagsyncSiteConfigurations =
clusterConfig.getProperties().get("ranger-tagsync-site");
+ assertNull(updatedRangerTagsyncSiteConfigurations);
+ }
+
+ @Test
+ public void
testDoUpdateForClusterWithNameNodeFederationEnabledTagsyncEnabledPluginEnabledNoTagsync()
throws Exception {
+ final String expectedNameService = "mynameservice";
+ final String expectedNameServiceTwo = "mynameservicetwo";
+ final String expectedHostName = "c6401.apache.ambari.org";
+ final String expectedHostNameTwo = "c6402.apache.ambari.org";
+ final String expectedHostNameThree = "c6403.apache.ambari.org";
+ final String expectedHostNameFour = "c6404.apache.ambari.org";
+ final String expectedPortNum = "808080";
+ final String expectedNodeOne = "nn1";
+ final String expectedNodeTwo = "nn2";
+ final String expectedNodeThree = "nn3";
+ final String expectedNodeFour = "nn4";
+ final String expectedHostGroupName = "host_group_1";
+ final String expectedHostGroupNameTwo = "host_group_2";
+ final String expectedHostGroupNameThree = "host-group-3";
+ final String expectedHostGroupNameFour = "host-group-4";
+
+ Map<String, Map<String, String>> properties = new HashMap<>();
+
+ Map<String, String> hdfsSiteProperties = new HashMap<>();
+ Map<String, String> hbaseSiteProperties = new HashMap<>();
+ Map<String, String> hadoopEnvProperties = new HashMap<>();
+ Map<String, String> coreSiteProperties = new HashMap<>();
+ Map<String, String> accumuloSiteProperties = new HashMap<>();
+ Map<String, String> rangerHDFSPluginProperties = new HashMap<>();
+ Map<String, String> rangerHDFSSecurityProperties = new HashMap<>();
+
+ properties.put("hdfs-site", hdfsSiteProperties);
+ properties.put("hadoop-env", hadoopEnvProperties);
+ properties.put("core-site", coreSiteProperties);
+ properties.put("hbase-site", hbaseSiteProperties);
+ properties.put("accumulo-site", accumuloSiteProperties);
+ properties.put("ranger-hdfs-plugin-properties",
rangerHDFSPluginProperties);
+ properties.put("ranger-hdfs-security", rangerHDFSSecurityProperties);
+
+ // setup multiple nameservices, to indicate NameNode Federation will be
used
+ hdfsSiteProperties.put("dfs.nameservices", expectedNameService + "," +
expectedNameServiceTwo);
+ hdfsSiteProperties.put("dfs.ha.namenodes.mynameservice", expectedNodeOne +
", " + expectedNodeTwo);
+ hdfsSiteProperties.put("dfs.ha.namenodes." + expectedNameServiceTwo,
expectedNodeThree + "," + expectedNodeFour);
+
+ //setup nameservice-specific properties
+ hdfsSiteProperties.put("dfs.namenode.shared.edits.dir" + "." +
expectedNameService,
+ "qjournal://" + createExportedAddress(expectedPortNum,
expectedHostGroupName) + ";" + createExportedAddress(expectedPortNum,
expectedHostGroupNameTwo) + "/ns1");
+ hdfsSiteProperties.put("dfs.namenode.shared.edits.dir" + "." +
expectedNameServiceTwo,
+ "qjournal://" + createExportedAddress(expectedPortNum,
expectedHostGroupName) + ";" + createExportedAddress(expectedPortNum,
expectedHostGroupNameTwo) + "/ns2");
+
+
+ // setup properties that include exported host group information
+ hdfsSiteProperties.put("dfs.namenode.https-address." + expectedNameService
+ "." + expectedNodeOne, createExportedAddress(expectedPortNum,
expectedHostGroupName));
+ hdfsSiteProperties.put("dfs.namenode.https-address." + expectedNameService
+ "." + expectedNodeTwo, createExportedAddress(expectedPortNum,
expectedHostGroupNameTwo));
+ hdfsSiteProperties.put("dfs.namenode.http-address." + expectedNameService
+ "." + expectedNodeOne, createExportedAddress(expectedPortNum,
expectedHostGroupName));
+ hdfsSiteProperties.put("dfs.namenode.http-address." + expectedNameService
+ "." + expectedNodeTwo, createExportedAddress(expectedPortNum,
expectedHostGroupNameTwo));
+ hdfsSiteProperties.put("dfs.namenode.rpc-address." + expectedNameService +
"." + expectedNodeOne, createExportedAddress(expectedPortNum,
expectedHostGroupName));
+ hdfsSiteProperties.put("dfs.namenode.rpc-address." + expectedNameService +
"." + expectedNodeTwo, createExportedAddress(expectedPortNum,
expectedHostGroupNameTwo));
+ hdfsSiteProperties.put("dfs.namenode.servicerpc-address." +
expectedNameService + "." + expectedNodeOne,
createExportedAddress(expectedPortNum, expectedHostGroupName));
+ hdfsSiteProperties.put("dfs.namenode.servicerpc-address." +
expectedNameService + "." + expectedNodeTwo,
createExportedAddress(expectedPortNum, expectedHostGroupNameTwo));
+
+ hdfsSiteProperties.put("dfs.namenode.https-address." +
expectedNameServiceTwo + "." + expectedNodeThree,
createExportedAddress(expectedPortNum, expectedHostGroupNameThree));
+ hdfsSiteProperties.put("dfs.namenode.https-address." +
expectedNameServiceTwo + "." + expectedNodeFour,
createExportedAddress(expectedPortNum, expectedHostGroupNameFour));
+ hdfsSiteProperties.put("dfs.namenode.http-address." +
expectedNameServiceTwo + "." + expectedNodeThree,
createExportedAddress(expectedPortNum, expectedHostGroupNameThree));
+ hdfsSiteProperties.put("dfs.namenode.http-address." +
expectedNameServiceTwo + "." + expectedNodeFour,
createExportedAddress(expectedPortNum, expectedHostGroupNameFour));
+ hdfsSiteProperties.put("dfs.namenode.rpc-address." +
expectedNameServiceTwo + "." + expectedNodeThree,
createExportedAddress(expectedPortNum, expectedHostGroupNameThree));
+ hdfsSiteProperties.put("dfs.namenode.rpc-address." +
expectedNameServiceTwo + "." + expectedNodeFour,
createExportedAddress(expectedPortNum, expectedHostGroupNameFour));
+ hdfsSiteProperties.put("dfs.namenode.servicerpc-address." +
expectedNameServiceTwo + "." + expectedNodeThree,
createExportedAddress(expectedPortNum, expectedHostGroupNameThree));
+ hdfsSiteProperties.put("dfs.namenode.servicerpc-address." +
expectedNameServiceTwo + "." + expectedNodeFour,
createExportedAddress(expectedPortNum, expectedHostGroupNameFour));
+
+
+ // add properties that require the SECONDARY_NAMENODE, which
+ // is not included in this test
+ hdfsSiteProperties.put("dfs.secondary.http.address", "localhost:8080");
+ hdfsSiteProperties.put("dfs.namenode.secondary.http-address",
"localhost:8080");
+
+
+ // add properties that are used in non-HA HDFS NameNode settings
+ // to verify that these are eventually removed by the filter
+ hdfsSiteProperties.put("dfs.namenode.http-address", "localhost:8080");
+ hdfsSiteProperties.put("dfs.namenode.https-address", "localhost:8081");
+ hdfsSiteProperties.put("dfs.namenode.rpc-address", "localhost:8082");
+
+ // configure the defaultFS to use the nameservice URL
+ coreSiteProperties.put("fs.defaultFS", "hdfs://" + expectedNameService);
+
+ // configure the hbase rootdir to use the nameservice URL
+ hbaseSiteProperties.put("hbase.rootdir", "hdfs://" + expectedNameService +
"/hbase/test/root/dir");
+
+ // configure the hbase rootdir to use the nameservice URL
+ accumuloSiteProperties.put("instance.volumes", "hdfs://" +
expectedNameService + "/accumulo/test/instance/volumes");
+
+ rangerHDFSPluginProperties.put("hadoop.rpc.protection","authentication");
+ rangerHDFSPluginProperties.put("REPOSITORY_CONFIG_USERNAME","hadoop");
+ rangerHDFSPluginProperties.put("REPOSITORY_CONFIG_PASSWORD","hadoop");
+ rangerHDFSPluginProperties.put("ranger-hdfs-plugin-enabled","Yes");
+
+
rangerHDFSSecurityProperties.put("ranger.plugin.hdfs.service.name","{{repo_name}}");
+
+ Configuration clusterConfig = new Configuration(properties, emptyMap());
+
+ Collection<String> hgComponents = new HashSet<>();
+ hgComponents.add("NAMENODE");
+ hgComponents.add("RANGER_ADMIN");
+ hgComponents.add("RANGER_USERSYNC");
+ TestHostGroup group1 = new TestHostGroup(expectedHostGroupName,
hgComponents, Collections.singleton(expectedHostName));
+
+ Collection<String> hgComponents2 = new HashSet<>();
+ hgComponents2.add("NAMENODE");
+ hgComponents2.add("ATLAS_SERVER");
+ hgComponents2.add("ATLAS_CLIENT");
+ TestHostGroup group2 = new TestHostGroup(expectedHostGroupNameTwo,
hgComponents2, Collections.singleton(expectedHostNameTwo));
+
+ // add third and fourth hostgroup with NAMENODE, to simulate HDFS NameNode
Federation
+ TestHostGroup group3 = new TestHostGroup(expectedHostGroupNameThree,
Collections.singleton("NAMENODE"),
Collections.singleton(expectedHostNameThree));
+ TestHostGroup group4 = new TestHostGroup(expectedHostGroupNameFour,
Collections.singleton("NAMENODE"), Collections.singleton(expectedHostNameFour));
+
+ Collection<TestHostGroup> hostGroups = new ArrayList<>();
+ hostGroups.add(group1);
+ hostGroups.add(group2);
+ hostGroups.add(group3);
+ hostGroups.add(group4);
+
+ expect(stack.getCardinality("NAMENODE")).andReturn(new
Cardinality("1-2")).anyTimes();
+ expect(stack.getCardinality("SECONDARY_NAMENODE")).andReturn(new
Cardinality("1")).anyTimes();
+
+ ClusterTopology topology = createClusterTopology(bp, clusterConfig,
hostGroups);
+ BlueprintConfigurationProcessor updater = new
BlueprintConfigurationProcessor(topology);
+
+ Set<String> updatedConfigTypes = updater.doUpdateForClusterCreate();
+
+ // verify that correct configuration types were listed as updated in the
returned set
+ assertEquals(ImmutableSet.of("cluster-env", "hdfs-site"),
updatedConfigTypes);
+
+
+ Map<String, String> updatedRangerTagsyncSiteConfigurations =
clusterConfig.getProperties().get("ranger-tagsync-site");
+ assertNull(updatedRangerTagsyncSiteConfigurations);
+ }
+
+
private static String createExportedAddress(String expectedPortNum, String
expectedHostGroupName) {
return createExportedHostName(expectedHostGroupName, expectedPortNum);
}
--
To stop receiving notification emails like this one, please contact
[email protected].
