Repository: ambari Updated Branches: refs/heads/trunk 714d2535e -> 832f7e1ec
AMBARI-11308. Oozie HA configuration property not exported properly in a Blueprint. (rnettleton) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/832f7e1e Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/832f7e1e Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/832f7e1e Branch: refs/heads/trunk Commit: 832f7e1ec7a04c8ade400f9bfa679d2d7069ae13 Parents: 714d253 Author: Bob Nettleton <[email protected]> Authored: Fri May 22 11:17:10 2015 -0400 Committer: Bob Nettleton <[email protected]> Committed: Fri May 22 11:17:31 2015 -0400 ---------------------------------------------------------------------- .../BlueprintConfigurationProcessor.java | 70 +++++++++++++++++++ .../OOZIE/5.0.0.2.3/metainfo.xml | 2 +- .../BlueprintConfigurationProcessorTest.java | 71 ++++++++++++++++++++ 3 files changed, 142 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/832f7e1e/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java index 6984881..a84bf3d 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java @@ -231,6 +231,10 @@ public class BlueprintConfigurationProcessor { doYarnResourceManagerHAUpdate(); } + if (isOozieServerHAEnabled(clusterTopology.getConfiguration().getFullProperties())) { + doOozieServerHAUpdate(); + } + Collection<Map<String, Map<String, String>>> allConfigs = new ArrayList<Map<String, Map<String, String>>>(); allConfigs.add(clusterTopology.getConfiguration().getFullProperties()); for (HostGroupInfo groupInfo : clusterTopology.getHostGroupInfo().values()) { @@ -304,6 +308,10 @@ public class BlueprintConfigurationProcessor { updaters = addYarnResourceManagerHAUpdaters(updaters); } + if (isOozieServerHAEnabled(clusterTopology.getConfiguration().getFullProperties())) { + updaters = addOozieServerHAUpdaters(updaters); + } + return updaters; } @@ -359,6 +367,33 @@ public class BlueprintConfigurationProcessor { return highAvailabilityUpdaters; } + + /** + * Creates a Collection of PropertyUpdater maps that include the OozieServer HA properties, and + * adds these to the list of updaters used to process the cluster configuration. + + * This new Collection includes the statically-defined updaters, + * in addition to the HA-related updaters. + * + * @param updaters a Collection of updater maps to be included in the list of updaters for + * this cluster config update + * @return A Collection of PropertyUpdater maps to handle the cluster config update + */ + private Collection<Map<String, Map<String, PropertyUpdater>>> addOozieServerHAUpdaters(Collection<Map<String, Map<String, PropertyUpdater>>> updaters) { + Collection<Map<String, Map<String, PropertyUpdater>>> highAvailabilityUpdaters = + new LinkedList<Map<String, Map<String, PropertyUpdater>>>(); + + // always add the statically-defined list of updaters to the list to use + // in processing cluster configuration + highAvailabilityUpdaters.addAll(updaters); + + // add the updaters for the Oozie HA properties not defined in stack, but + // required to be present/updated in oozie-site + highAvailabilityUpdaters.add(createMapOfOozieServerHAUpdaters()); + + return highAvailabilityUpdaters; + } + /** * Performs export update for the set of properties that do not * require update during cluster setup, but should be removed @@ -418,6 +453,20 @@ public class BlueprintConfigurationProcessor { } } + /** + * Perform export update processing for HA configuration for Oozie servers. The properties used + * in Oozie HA are not defined in the stack, but need to be added at runtime during an HA + * deployment in order to support exporting/redeploying clusters with Oozie HA config. + * + */ + public void doOozieServerHAUpdate() { + Map<String, Map<String, PropertyUpdater>> highAvailabilityUpdaters = createMapOfOozieServerHAUpdaters(); + + if (highAvailabilityUpdaters.get("oozie-site").size() > 0) { + doMultiHostExportUpdate(highAvailabilityUpdaters, clusterTopology.getConfiguration().getFullProperties()); + } + } + /** * Creates map of PropertyUpdater instances that are associated with @@ -481,6 +530,27 @@ public class BlueprintConfigurationProcessor { } /** + * Creates map of PropertyUpdater instances that are associated with + * Oozie Server High Availability (HA). + * + * @return a Map of registered PropertyUpdaters for handling HA properties in oozie-site + */ + private Map<String, Map<String, PropertyUpdater>> createMapOfOozieServerHAUpdaters() { + Map<String, Map<String, PropertyUpdater>> highAvailabilityUpdaters = new HashMap<String, Map<String, PropertyUpdater>>(); + Map<String, PropertyUpdater> oozieSiteUpdatersForAvailability = new HashMap<String, PropertyUpdater>(); + highAvailabilityUpdaters.put("oozie-site", oozieSiteUpdatersForAvailability); + + // register a multi-host property updater for this Oozie property. + // this property is not defined in the stacks, since HA is not supported yet + // by the stack definition syntax. This property should only be considered in + // an Oozie HA cluster. + oozieSiteUpdatersForAvailability.put("oozie.zookeeper.connection.string", new MultipleHostTopologyUpdater("ZOOKEEPER_SERVER")); + + return highAvailabilityUpdaters; + + } + + /** * Static convenience function to determine if Oozie HA is enabled * @param configProperties configuration properties for this cluster * @return true if Oozie HA is enabled http://git-wip-us.apache.org/repos/asf/ambari/blob/832f7e1e/ambari-server/src/main/resources/common-services/OOZIE/5.0.0.2.3/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/OOZIE/5.0.0.2.3/metainfo.xml b/ambari-server/src/main/resources/common-services/OOZIE/5.0.0.2.3/metainfo.xml index 463abc3..a78c93a 100644 --- a/ambari-server/src/main/resources/common-services/OOZIE/5.0.0.2.3/metainfo.xml +++ b/ambari-server/src/main/resources/common-services/OOZIE/5.0.0.2.3/metainfo.xml @@ -30,7 +30,7 @@ <name>OOZIE_SERVER</name> <displayName>Oozie Server</displayName> <category>MASTER</category> - <cardinality>1</cardinality> + <cardinality>1+</cardinality> <versionAdvertised>true</versionAdvertised> <dependencies> <dependency> http://git-wip-us.apache.org/repos/asf/ambari/blob/832f7e1e/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java index 6849b45..595cc4a 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java @@ -2223,6 +2223,77 @@ public class BlueprintConfigurationProcessorTest { } @Test + public void testOozieHAEnabledExport() throws Exception { + final String expectedHostName = "c6401.apache.ambari.org"; + final String expectedHostNameTwo = "c6402.ambari.apache.org"; + final String expectedExternalHost = "c6408.ambari.apache.org"; + final String expectedHostGroupName = "host_group_1"; + final String expectedHostGroupNameTwo = "host_group_2"; + final String expectedPortNum = "80000"; + + Map<String, Map<String, String>> configProperties = new HashMap<String, Map<String, String>>(); + Map<String, String> oozieSiteProperties = new HashMap<String, String>(); + Map<String, String> oozieEnvProperties = new HashMap<String, String>(); + Map<String, String> coreSiteProperties = new HashMap<String, String>(); + + configProperties.put("oozie-site", oozieSiteProperties); + configProperties.put("oozie-env", oozieEnvProperties); + configProperties.put("hive-env", oozieEnvProperties); + configProperties.put("core-site", coreSiteProperties); + + + oozieSiteProperties.put("oozie.base.url", expectedHostName + ":" + expectedPortNum); + oozieSiteProperties.put("oozie.authentication.kerberos.principal", expectedHostName); + oozieSiteProperties.put("oozie.service.HadoopAccessorService.kerberos.principal", expectedHostName); + oozieSiteProperties.put("oozie.service.JPAService.jdbc.url", "jdbc:mysql://" + expectedExternalHost + "/ooziedb"); + + // simulate the Oozie HA configuration + oozieSiteProperties.put("oozie.services.ext", + "org.apache.oozie.service.ZKLocksService,org.apache.oozie.service.ZKXLogStreamingService,org.apache.oozie.service.ZKJobsConcurrencyService,org.apache.oozie.service.ZKUUIDService"); + oozieSiteProperties.put("oozie.zookeeper.connection.string", createHostAddress(expectedHostName, "2181") + "," + createHostAddress(expectedHostNameTwo, "2181")); + + + oozieEnvProperties.put("oozie_hostname", expectedHostName); + oozieEnvProperties.put("oozie_existing_mysql_host", expectedExternalHost); + + coreSiteProperties.put("hadoop.proxyuser.oozie.hosts", expectedHostName + "," + expectedHostNameTwo); + + Configuration clusterConfig = new Configuration(configProperties, Collections.<String, Map<String, Map<String, String>>>emptyMap()); + Collection<String> hgComponents = new HashSet<String>(); + hgComponents.add("OOZIE_SERVER"); + hgComponents.add("ZOOKEEPER_SERVER"); + TestHostGroup group1 = new TestHostGroup(expectedHostGroupName, hgComponents, Collections.singleton(expectedHostName)); + + Collection<String> hgComponents2 = new HashSet<String>(); + hgComponents2.add("OOZIE_SERVER"); + hgComponents2.add("ZOOKEEPER_SERVER"); + TestHostGroup group2 = new TestHostGroup(expectedHostGroupNameTwo, hgComponents2, Collections.singleton(expectedHostNameTwo)); + + Collection<TestHostGroup> hostGroups = new HashSet<TestHostGroup>(); + hostGroups.add(group1); + hostGroups.add(group2); + + expect(stack.getCardinality("OOZIE_SERVER")).andReturn(new Cardinality("1+")).anyTimes(); + + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); + updater.doUpdateForBlueprintExport(); + + assertEquals("oozie property not updated correctly", + createExportedHostName(expectedHostGroupName, expectedPortNum), oozieSiteProperties.get("oozie.base.url")); + assertEquals("oozie property not updated correctly", + createExportedHostName(expectedHostGroupName), oozieSiteProperties.get("oozie.authentication.kerberos.principal")); + assertEquals("oozie property not updated correctly", + createExportedHostName(expectedHostGroupName), oozieSiteProperties.get("oozie.service.HadoopAccessorService.kerberos.principal")); + assertEquals("oozie property not updated correctly", + createExportedHostName(expectedHostGroupName), oozieEnvProperties.get("oozie_hostname")); + assertEquals("oozie property not updated correctly", + createExportedHostName(expectedHostGroupName) + "," + createExportedHostName(expectedHostGroupNameTwo), coreSiteProperties.get("hadoop.proxyuser.oozie.hosts")); + assertEquals("oozie property not updated correctly", + createExportedAddress("2181", expectedHostGroupName) + "," + createExportedAddress("2181", expectedHostGroupNameTwo), oozieSiteProperties.get("oozie.zookeeper.connection.string")); + } + + @Test public void testYarnHighAvailabilityConfigClusterUpdateSpecifyingHostNamesDirectly() throws Exception { final String expectedHostName = "c6401.apache.ambari.org"; final String expectedPortNum = "808080";
