Repository: ambari Updated Branches: refs/heads/trunk a1cd22f99 -> 90da27509
AMBARI-10388. Blueprint not properly handling host group names that include "-" characters. (rnettleton) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/90da2750 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/90da2750 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/90da2750 Branch: refs/heads/trunk Commit: 90da27509c93825eee1db57a5e962ab31f0cc550 Parents: a1cd22f Author: Bob Nettleton <[email protected]> Authored: Tue Apr 7 18:49:29 2015 -0400 Committer: Bob Nettleton <[email protected]> Committed: Tue Apr 7 18:49:29 2015 -0400 ---------------------------------------------------------------------- .../BlueprintConfigurationProcessor.java | 3 +- .../BlueprintConfigurationProcessorTest.java | 162 +++++++++++++++++++ 2 files changed, 164 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/90da2750/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java index 04027af..12768f7 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java @@ -85,7 +85,7 @@ public class BlueprintConfigurationProcessor { /** * Compiled regex for hostgroup token with port information. */ - private static Pattern HOSTGROUP_PORT_REGEX = Pattern.compile("%HOSTGROUP::(\\w+|\\d+)%:?(\\d+)?"); + private static Pattern HOSTGROUP_PORT_REGEX = Pattern.compile("%HOSTGROUP::(\\S+?)%:?(\\d+)?"); /** * Statically-defined set of properties that can support using a nameservice name @@ -95,6 +95,7 @@ public class BlueprintConfigurationProcessor { new HashSet<String>(Arrays.asList("fs.defaultFS", "hbase.rootdir")); + /** * Configuration properties to be updated */ http://git-wip-us.apache.org/repos/asf/ambari/blob/90da2750/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java index d8794ac..b886956 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java @@ -788,6 +788,62 @@ public class BlueprintConfigurationProcessorTest { } @Test + public void testDoUpdateForClusterCreate_SingleHostProperty__exportedValue_UsingMinusSymbolInHostGroupName() { + Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>(); + Map<String, String> typeProps = new HashMap<String, String>(); + typeProps.put("yarn.resourcemanager.hostname", "%HOSTGROUP::os-amb-r6-secha-1427972156-hbaseha-3-6%"); + properties.put("yarn-site", typeProps); + + Collection<String> hgComponents = new HashSet<String>(); + hgComponents.add("NAMENODE"); + hgComponents.add("SECONDARY_NAMENODE"); + hgComponents.add("RESOURCEMANAGER"); + HostGroup group1 = new TestHostGroup("os-amb-r6-secha-1427972156-hbaseha-3-6", Collections.singleton("testhost"), hgComponents); + + Collection<String> hgComponents2 = new HashSet<String>(); + hgComponents2.add("DATANODE"); + hgComponents2.add("HDFS_CLIENT"); + HostGroup group2 = new TestHostGroup("group2", Collections.singleton("testhost2"), hgComponents2); + + Map<String, HostGroup> hostGroups = new HashMap<String, HostGroup>(); + hostGroups.put(group1.getName(), group1); + hostGroups.put(group2.getName(), group2); + + BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(properties); + Map<String, Map<String, String>> updatedProperties = updater.doUpdateForClusterCreate(hostGroups, null); + String updatedVal = updatedProperties.get("yarn-site").get("yarn.resourcemanager.hostname"); + assertEquals("testhost", updatedVal); + } + + @Test + public void testDoUpdateForClusterCreate_SingleHostProperty__exportedValue_WithPort_UsingMinusSymbolInHostGroupName() { + Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>(); + Map<String, String> typeProps = new HashMap<String, String>(); + typeProps.put("yarn.resourcemanager.hostname", "%HOSTGROUP::os-amb-r6-secha-1427972156-hbaseha-3-6%:2180"); + properties.put("yarn-site", typeProps); + + Collection<String> hgComponents = new HashSet<String>(); + hgComponents.add("NAMENODE"); + hgComponents.add("SECONDARY_NAMENODE"); + hgComponents.add("RESOURCEMANAGER"); + HostGroup group1 = new TestHostGroup("os-amb-r6-secha-1427972156-hbaseha-3-6", Collections.singleton("testhost"), hgComponents); + + Collection<String> hgComponents2 = new HashSet<String>(); + hgComponents2.add("DATANODE"); + hgComponents2.add("HDFS_CLIENT"); + HostGroup group2 = new TestHostGroup("group2", Collections.singleton("testhost2"), hgComponents2); + + Map<String, HostGroup> hostGroups = new HashMap<String, HostGroup>(); + hostGroups.put(group1.getName(), group1); + hostGroups.put(group2.getName(), group2); + + BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(properties); + Map<String, Map<String, String>> updatedProperties = updater.doUpdateForClusterCreate(hostGroups, null); + String updatedVal = updatedProperties.get("yarn-site").get("yarn.resourcemanager.hostname"); + assertEquals("testhost:2180", updatedVal); + } + + @Test public void testDoUpdateForClusterCreate_SingleHostProperty__exportedValue__WithPort() { Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>(); Map<String, String> typeProps = new HashMap<String, String>(); @@ -924,6 +980,62 @@ public class BlueprintConfigurationProcessorTest { } @Test + public void testDoUpdateForClusterCreate_MultiHostProperty__exportedValues___withPorts_UsingMinusSymbolInHostGroupName() { + Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>(); + Map<String, String> typeProps = new HashMap<String, String>(); + typeProps.put("ha.zookeeper.quorum", "%HOSTGROUP::os-amb-r6-secha-1427972156-hbaseha-3-6%:2181,%HOSTGROUP::os-amb-r6-secha-1427972156-hbaseha-3-5%:2181,%HOSTGROUP::os-amb-r6-secha-1427972156-hbaseha-3-7%:2181"); + properties.put("core-site", typeProps); + + Collection<String> hgComponents = new HashSet<String>(); + hgComponents.add("NAMENODE"); + hgComponents.add("SECONDARY_NAMENODE"); + hgComponents.add("ZOOKEEPER_SERVER"); + HostGroup group1 = new TestHostGroup("os-amb-r6-secha-1427972156-hbaseha-3-6", Collections.singleton("testhost"), hgComponents); + + Collection<String> hgComponents2 = new HashSet<String>(); + hgComponents2.add("DATANODE"); + hgComponents2.add("HDFS_CLIENT"); + hgComponents2.add("ZOOKEEPER_SERVER"); + Set<String> hosts2 = new HashSet<String>(); + hosts2.add("testhost2"); + hosts2.add("testhost2a"); + hosts2.add("testhost2b"); + HostGroup group2 = new TestHostGroup("os-amb-r6-secha-1427972156-hbaseha-3-5", hosts2, hgComponents2); + + Collection<String> hgComponents3 = new HashSet<String>(); + hgComponents2.add("HDFS_CLIENT"); + hgComponents2.add("ZOOKEEPER_CLIENT"); + Set<String> hosts3 = new HashSet<String>(); + hosts3.add("testhost3"); + hosts3.add("testhost3a"); + HostGroup group3 = new TestHostGroup("os-amb-r6-secha-1427972156-hbaseha-3-7", hosts3, hgComponents3); + + Map<String, HostGroup> hostGroups = new HashMap<String, HostGroup>(); + hostGroups.put(group1.getName(), group1); + hostGroups.put(group2.getName(), group2); + hostGroups.put(group3.getName(), group3); + + BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(properties); + Map<String, Map<String, String>> updatedProperties = updater.doUpdateForClusterCreate(hostGroups, null); + String updatedVal = updatedProperties.get("core-site").get("ha.zookeeper.quorum"); + String[] hosts = updatedVal.split(","); + + Collection<String> expectedHosts = new HashSet<String>(); + expectedHosts.add("testhost:2181"); + expectedHosts.add("testhost2:2181"); + expectedHosts.add("testhost2a:2181"); + expectedHosts.add("testhost2b:2181"); + expectedHosts.add("testhost3:2181"); + expectedHosts.add("testhost3a:2181"); + + assertEquals(6, hosts.length); + for (String host : hosts) { + assertTrue("Expected host :" + host + "was not included in the multi-server list in this property.", expectedHosts.contains(host)); + expectedHosts.remove(host); + } + } + + @Test public void testDoUpdateForClusterCreate_MultiHostProperty_exportedValues_withPorts_singleHostValue() { Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>(); Map<String, String> yarnSiteConfig = new HashMap<String, String>(); @@ -2307,6 +2419,56 @@ public class BlueprintConfigurationProcessorTest { } @Test + public void testHDFSConfigClusterUpdateQuorumJournalURL_UsingMinusSymbolInHostName() throws Exception { + final String expectedHostNameOne = "c6401.apache.ambari.org"; + final String expectedHostNameTwo = "c6402.apache.ambari.org"; + final String expectedPortNum = "808080"; + final String expectedHostGroupName = "host-group-1"; + final String expectedHostGroupNameTwo = "host-group-2"; + + EasyMockSupport mockSupport = new EasyMockSupport(); + + HostGroup mockHostGroupOne = mockSupport.createMock(HostGroup.class); + HostGroup mockHostGroupTwo = mockSupport.createMock(HostGroup.class); + + expect(mockHostGroupOne.getHostInfo()).andReturn(Arrays.asList(expectedHostNameOne)).atLeastOnce(); + expect(mockHostGroupTwo.getHostInfo()).andReturn(Arrays.asList(expectedHostNameTwo)).atLeastOnce(); + + mockSupport.replayAll(); + + Map<String, Map<String, String>> configProperties = + new HashMap<String, Map<String, String>>(); + + Map<String, String> hdfsSiteProperties = + new HashMap<String, String>(); + + configProperties.put("hdfs-site", hdfsSiteProperties); + + // setup properties that include host information + // setup shared edit property, that includes a qjournal URL scheme + hdfsSiteProperties.put("dfs.namenode.shared.edits.dir", "qjournal://" + createExportedAddress(expectedPortNum, expectedHostGroupName) + ";" + createExportedAddress(expectedPortNum, expectedHostGroupNameTwo) + "/mycluster"); + + BlueprintConfigurationProcessor configProcessor = + new BlueprintConfigurationProcessor(configProperties); + + Map<String, HostGroup> mapOfHostGroups = + new HashMap<String, HostGroup>(); + mapOfHostGroups.put(expectedHostGroupName, mockHostGroupOne); + mapOfHostGroups.put(expectedHostGroupNameTwo, mockHostGroupTwo); + + // call top-level export method + configProcessor.doUpdateForClusterCreate(mapOfHostGroups, null); + + // expect that all servers are included in the updated config, and that the qjournal URL format is preserved + assertEquals("HDFS HA shared edits directory property not properly updated for cluster create.", + "qjournal://" + createHostAddress(expectedHostNameOne, expectedPortNum) + ";" + createHostAddress(expectedHostNameTwo, expectedPortNum) + "/mycluster", + hdfsSiteProperties.get("dfs.namenode.shared.edits.dir")); + + mockSupport.verifyAll(); + + } + + @Test public void testHiveConfigExported() throws Exception { final String expectedHostName = "c6401.apache.ambari.org"; final String expectedHostNameTwo = "c6402.ambari.apache.org";
