AMBARI-14745. Ambari server throws error when unused configs are present in blueprint. (Oliver Szabo via rnettleton)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/88dab3b4 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/88dab3b4 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/88dab3b4 Branch: refs/heads/2.2.1-maint Commit: 88dab3b48e8067613e12a1972742d820eb7b48ed Parents: 4c31108 Author: Bob Nettleton <rnettle...@hortonworks.com> Authored: Fri Feb 5 11:44:23 2016 -0500 Committer: Bob Nettleton <rnettle...@hortonworks.com> Committed: Fri Feb 5 11:44:23 2016 -0500 ---------------------------------------------------------------------- .../topology/ClusterConfigurationRequest.java | 28 ++++++++ .../ambari/server/topology/Configuration.java | 15 ++++ .../ClusterConfigurationRequestTest.java | 75 +++++++++++++++++--- .../ClusterInstallWithoutStartTest.java | 2 + .../server/topology/ConfigurationTest.java | 14 ++++ .../server/topology/TopologyManagerTest.java | 2 + 6 files changed, 128 insertions(+), 8 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/88dab3b4/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterConfigurationRequest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterConfigurationRequest.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterConfigurationRequest.java index 464aee7..a995a3b 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterConfigurationRequest.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterConfigurationRequest.java @@ -30,6 +30,7 @@ import org.apache.ambari.server.serveraction.kerberos.KerberosInvalidConfigurati import org.apache.ambari.server.state.Cluster; import org.apache.ambari.server.state.SecurityType; import org.apache.ambari.server.utils.StageUtils; +import org.apache.commons.collections.MapUtils; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -74,11 +75,38 @@ public class ClusterConfigurationRequest { // set initial configuration (not topology resolved) this.configurationProcessor = new BlueprintConfigurationProcessor(clusterTopology); this.stackAdvisorBlueprintProcessor = stackAdvisorBlueprintProcessor; + removeOrphanConfigTypes(clusterTopology); if (setInitial) { setConfigurationsOnCluster(clusterTopology, TopologyManager.INITIAL_CONFIG_TAG, Collections.<String>emptySet()); } } + /** + * Remove config-types, if there is no any services related to them (except cluster-env and global). + */ + private void removeOrphanConfigTypes(ClusterTopology clusterTopology) { + Configuration configuration = clusterTopology.getConfiguration(); + Collection<String> configTypes = configuration.getAllConfigTypes(); + for (String configType : configTypes) { + if (!configType.equals("cluster-env") && !configType.equals("global")) { + String service = clusterTopology.getBlueprint().getStack().getServiceForConfigType(configType); + if (!clusterTopology.getBlueprint().getServices().contains(service)) { + configuration.removeConfigType(configType); + LOG.info("Not found any service for config type '{}'. It will be removed from configuration.", configType); + Map<String, HostGroupInfo> hostGroupInfoMap = clusterTopology.getHostGroupInfo(); + if (MapUtils.isNotEmpty(hostGroupInfoMap)) { + for (Map.Entry<String, HostGroupInfo> hostGroupInfo : hostGroupInfoMap.entrySet()) { + if (hostGroupInfo.getValue().getConfiguration() != null) { + hostGroupInfo.getValue().getConfiguration().removeConfigType(configType); + LOG.info("Not found any service for config type '{}'. It will be removed from host group scoped configuration.", configType); + } + } + } + } + } + } + } + public ClusterConfigurationRequest(AmbariContext ambariContext, ClusterTopology topology, boolean setInitial, StackAdvisorBlueprintProcessor stackAdvisorBlueprintProcessor, boolean configureSecurity) { this(ambariContext, topology, setInitial, stackAdvisorBlueprintProcessor); this.configureSecurity = configureSecurity; http://git-wip-us.apache.org/repos/asf/ambari/blob/88dab3b4/ambari-server/src/main/java/org/apache/ambari/server/topology/Configuration.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/Configuration.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/Configuration.java index 108ff74..79281b4 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/Configuration.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/Configuration.java @@ -344,4 +344,19 @@ public class Configuration { public void setParentConfiguration(Configuration parent) { parentConfiguration = parent; } + + /** + * Remove all occurrences of a config type + */ + public void removeConfigType(String configType) { + if (properties != null && properties.containsKey(configType)) { + properties.remove(configType); + } + if (attributes != null && attributes.containsKey(configType)) { + attributes.remove(configType); + } + if (parentConfiguration != null) { + parentConfiguration.removeConfigType(configType); + } + } } http://git-wip-us.apache.org/repos/asf/ambari/blob/88dab3b4/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterConfigurationRequestTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterConfigurationRequestTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterConfigurationRequestTest.java index 5967a64..3ed8ed5 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterConfigurationRequestTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterConfigurationRequestTest.java @@ -28,6 +28,17 @@ import org.apache.ambari.server.controller.internal.Stack; import org.apache.ambari.server.serveraction.kerberos.KerberosInvalidConfigurationException; import org.apache.ambari.server.state.Cluster; import org.apache.ambari.server.state.Clusters; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import com.google.common.collect.Maps; +import org.easymock.EasyMock; import org.easymock.Capture; import org.easymock.CaptureType; import org.easymock.EasyMockRule; @@ -40,14 +51,6 @@ import org.powermock.api.easymock.PowerMock; import org.powermock.core.classloader.annotations.PrepareForTest; import org.powermock.modules.junit4.PowerMockRunner; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; - import static org.easymock.EasyMock.anyBoolean; import static org.easymock.EasyMock.anyObject; import static org.easymock.EasyMock.anyString; @@ -55,6 +58,8 @@ import static org.easymock.EasyMock.expect; import static org.easymock.EasyMock.expectLastCall; import static org.easymock.EasyMock.newCapture; import static org.easymock.EasyMock.verify; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; import static org.easymock.EasyMock.capture; import static org.junit.Assert.assertEquals; @@ -183,6 +188,7 @@ public class ClusterConfigurationRequestTest { expect(clusters.getCluster("testCluster")).andReturn(cluster).anyTimes(); expect(blueprint.getStack()).andReturn(stack).anyTimes(); + expect(stack.getServiceForConfigType(anyString())).andReturn("KERBEROS").anyTimes(); expect(stack.getAllConfigurationTypes(anyString())).andReturn(Collections.<String>singletonList("testConfigType") ).anyTimes(); expect(stack.getExcludedConfigurationTypes(anyString())).andReturn(Collections.<String>emptySet()).anyTimes(); @@ -307,4 +313,57 @@ public class ClusterConfigurationRequestTest { } + @Test + public void testProcessClusterConfigRequestRemoveUnusedConfigTypes() { + // GIVEN + Configuration configuration = createConfigurations(); + Set<String> services = new HashSet<String>(); + services.add("HDFS"); + services.add("RANGER"); + Map<String, HostGroupInfo> hostGroupInfoMap = Maps.newHashMap(); + HostGroupInfo hg1 = new HostGroupInfo("hg1"); + hg1.setConfiguration(createConfigurations()); + hostGroupInfoMap.put("hg1", hg1); + + expect(topology.getConfiguration()).andReturn(configuration).anyTimes(); + expect(topology.getBlueprint()).andReturn(blueprint).anyTimes(); + expect(topology.getHostGroupInfo()).andReturn(hostGroupInfoMap); + expect(blueprint.getStack()).andReturn(stack).anyTimes(); + expect(blueprint.getServices()).andReturn(services).anyTimes(); + expect(stack.getServiceForConfigType("hdfs-site")).andReturn("HDFS").anyTimes(); + expect(stack.getServiceForConfigType("admin-properties")).andReturn("RANGER").anyTimes(); + expect(stack.getServiceForConfigType("yarn-site")).andReturn("YARN").anyTimes(); + + EasyMock.replay(stack, blueprint, topology); + // WHEN + new ClusterConfigurationRequest(ambariContext, topology, false, stackAdvisorBlueprintProcessor); + // THEN + assertFalse(configuration.getFullProperties().containsKey("yarn-site")); + assertFalse(configuration.getFullAttributes().containsKey("yarn-site")); + assertTrue(configuration.getFullAttributes().containsKey("hdfs-site")); + assertTrue(configuration.getFullProperties().containsKey("cluster-env")); + assertTrue(configuration.getFullProperties().containsKey("global")); + assertFalse(hg1.getConfiguration().getFullAttributes().containsKey("yarn-site")); + verify(stack, blueprint, topology); + } + + private Configuration createConfigurations() { + Map<String, Map<String, String>> firstLevelConfig = Maps.newHashMap(); + firstLevelConfig.put("hdfs-site", new HashMap<String, String>()); + firstLevelConfig.put("yarn-site", new HashMap<String, String>()); + firstLevelConfig.put("cluster-env", new HashMap<String, String>()); + firstLevelConfig.put("global", new HashMap<String, String>()); + + Map<String, Map<String, Map<String, String>>> firstLevelAttributes = Maps.newHashMap(); + firstLevelAttributes.put("hdfs-site", new HashMap<String, Map<String, String>>()); + + Map<String, Map<String, String>> secondLevelConfig = Maps.newHashMap(); + secondLevelConfig.put("admin-properties", new HashMap<String, String>()); + Map<String, Map<String, Map<String, String>>> secondLevelAttributes = Maps.newHashMap(); + secondLevelAttributes.put("admin-properties", new HashMap<String, Map<String, String>>()); + secondLevelAttributes.put("yarn-site", new HashMap<String, Map<String, String>>()); + + Configuration secondLevelConf = new Configuration(secondLevelConfig, secondLevelAttributes); + return new Configuration(firstLevelConfig, firstLevelAttributes, secondLevelConf); + } } http://git-wip-us.apache.org/repos/asf/ambari/blob/88dab3b4/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterInstallWithoutStartTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterInstallWithoutStartTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterInstallWithoutStartTest.java index dd66b1b..156580a 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterInstallWithoutStartTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterInstallWithoutStartTest.java @@ -251,6 +251,8 @@ public class ClusterInstallWithoutStartTest { expect(stack.getConfiguration()).andReturn(stackConfig).anyTimes(); expect(stack.getName()).andReturn(STACK_NAME).anyTimes(); expect(stack.getVersion()).andReturn(STACK_VERSION).anyTimes(); + expect(stack.getServiceForConfigType("service1-site")).andReturn("service1"); + expect(stack.getServiceForConfigType("service2-site")).andReturn("service2"); expect(stack.getExcludedConfigurationTypes("service1")).andReturn(Collections.<String>emptySet()).anyTimes(); expect(stack.getExcludedConfigurationTypes("service2")).andReturn(Collections.<String>emptySet()).anyTimes(); http://git-wip-us.apache.org/repos/asf/ambari/blob/88dab3b4/ambari-server/src/test/java/org/apache/ambari/server/topology/ConfigurationTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/ConfigurationTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/ConfigurationTest.java index e971e03..c4d70b5 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/topology/ConfigurationTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/ConfigurationTest.java @@ -346,6 +346,20 @@ public class ConfigurationTest { assertNull(configuration.getPropertyValue("type1", "XXXXX")); } + @Test + public void testRemoveConfigTypes() { + Configuration configuration = createConfigurationWithParents_PropsOnly(); + configuration.removeConfigType("type1"); + assertNull(configuration.getProperties().get("type1")); + } + + @Test + public void testRemoveConfigTypesForAttributes() { + Configuration configuration = createConfigurationWithParents_PropsOnly(); + configuration.removeConfigType("type1"); + assertNull(configuration.getAttributes().get("type1")); + } + private Configuration createConfigurationWithParents_PropsOnly() { // parents parent config properties Map<String, Map<String, String>> parentParentProperties = new HashMap<String, Map<String, String>>(); http://git-wip-us.apache.org/repos/asf/ambari/blob/88dab3b4/ambari-server/src/test/java/org/apache/ambari/server/topology/TopologyManagerTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/TopologyManagerTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/TopologyManagerTest.java index 7810f92..69c1935 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/topology/TopologyManagerTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/TopologyManagerTest.java @@ -228,6 +228,8 @@ public class TopologyManagerTest { expect(stack.getComponents()).andReturn(serviceComponents).anyTimes(); expect(stack.getComponents("service1")).andReturn(serviceComponents.get("service1")).anyTimes(); expect(stack.getComponents("service2")).andReturn(serviceComponents.get("service2")).anyTimes(); + expect(stack.getServiceForConfigType("service1-site")).andReturn("service1"); + expect(stack.getServiceForConfigType("service2-site")).andReturn("service2"); expect(stack.getConfiguration()).andReturn(stackConfig).anyTimes(); expect(stack.getName()).andReturn(STACK_NAME).anyTimes(); expect(stack.getVersion()).andReturn(STACK_VERSION).anyTimes();