Repository: ambari Updated Branches: refs/heads/branch-1.7.0 b941191c5 -> d9d9a5450 refs/heads/trunk a50ac9d13 -> abd9fcacb
AMBARI-7761. stale_configs parameter works incorrectly for Flume (dlysnichenko) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/abd9fcac Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/abd9fcac Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/abd9fcac Branch: refs/heads/trunk Commit: abd9fcacbf05c6fed8509ee689c9c342d28ce960 Parents: a50ac9d Author: Lisnichenko Dmitro <[email protected]> Authored: Tue Oct 14 16:32:09 2014 +0300 Committer: Lisnichenko Dmitro <[email protected]> Committed: Tue Oct 14 16:32:09 2014 +0300 ---------------------------------------------------------------------- .../ambari/server/state/ConfigHelper.java | 40 +++++++++-- .../api/util/StackExtensionHelperTest.java | 2 +- .../AmbariManagementControllerTest.java | 2 +- .../ambari/server/state/ConfigHelperTest.java | 73 ++++++++++++++++++-- .../services/FLUME/configuration/flume-conf.xml | 31 +++++++++ .../services/FLUME/configuration/flume-env.xml | 40 +++++++++++ .../FLUME/configuration/flume-log4j.xml | 31 +++++++++ .../HDP/2.0.6/services/FLUME/metainfo.xml | 69 ++++++++++++++++++ 8 files changed, 275 insertions(+), 13 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/abd9fcac/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java index 8b724b1..e15a62a 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java @@ -47,6 +47,7 @@ import org.apache.ambari.server.controller.ConfigurationRequest; import org.apache.ambari.server.orm.dao.ClusterDAO; import org.apache.ambari.server.orm.entities.ClusterConfigEntity; import org.apache.ambari.server.state.PropertyInfo.PropertyType; +import org.apache.ambari.server.state.configgroup.ConfigGroup; import org.apache.ambari.server.upgrade.UpgradeCatalog170; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -670,8 +671,8 @@ public class ConfigHelper { // desired and actual both define the type HostConfig hc = actual.get(type); Map<String, String> actualTags = buildTags(hc); - - if (!isTagChanged(tags, actualTags)) { + + if (!isTagChanged(tags, actualTags, hasGroupSpecificConfigsForType(cluster, sch.getHostName(), type))) { stale = false; } else if (type.equals(Configuration.GLOBAL_CONFIG_TAG)) { // tags are changed, need to find out what has changed, @@ -691,6 +692,31 @@ public class ConfigHelper { } /** + * Determines if the hostname has group specific configs for the type specified + * + * @param cluster + * @param hostname of the host to look for + * @param type the type to look for (e.g. flume-conf) + * @return <code>true</code> if the hostname has group specific configuration for the type + */ + private boolean hasGroupSpecificConfigsForType(Cluster cluster, String hostname, String type) { + try { + Map<Long, ConfigGroup> configGroups = cluster.getConfigGroupsByHostname(hostname); + if (configGroups != null && !configGroups.isEmpty()) { + for (ConfigGroup configGroup : configGroups.values()) { + Config config = configGroup.getConfigurations().get(type); + if (config != null) { + return true; + } + } + } + } catch (AmbariException ambariException) { + LOG.warn("Could not determine group configuration for host. Details: " + ambariException.getMessage()); + } + return false; + } + + /** * @return <code>true</code> if any service on the stack defines a property * for the type. */ @@ -761,10 +787,16 @@ public class ConfigHelper { /** * @return true if the tags are different in any way, even if not-specified */ - private boolean isTagChanged(Map<String, String> desiredTags, Map<String, String> actualTags) { - if (!actualTags.get(CLUSTER_DEFAULT_TAG).equals(desiredTags.get(CLUSTER_DEFAULT_TAG))) + private boolean isTagChanged(Map<String, String> desiredTags, Map<String, String> actualTags, boolean groupSpecificConfigs) { + if (!actualTags.get(CLUSTER_DEFAULT_TAG).equals(desiredTags.get(CLUSTER_DEFAULT_TAG)) && !groupSpecificConfigs) return true; + // if the host has group specific configs for type we should ignore the cluster level configs and compare specifics + if (groupSpecificConfigs) { + actualTags.remove(CLUSTER_DEFAULT_TAG); + desiredTags.remove(CLUSTER_DEFAULT_TAG); + } + Set<String> desiredSet = new HashSet<String>(desiredTags.values()); Set<String> actualSet = new HashSet<String>(actualTags.values()); http://git-wip-us.apache.org/repos/asf/ambari/blob/abd9fcac/ambari-server/src/test/java/org/apache/ambari/server/api/util/StackExtensionHelperTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/api/util/StackExtensionHelperTest.java b/ambari-server/src/test/java/org/apache/ambari/server/api/util/StackExtensionHelperTest.java index 84ba6aa..acd595d 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/api/util/StackExtensionHelperTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/api/util/StackExtensionHelperTest.java @@ -298,7 +298,7 @@ public class StackExtensionHelperTest { helper.populateServicesForStack(stackInfo); helper.fillInfo(); List<ServiceInfo> allServices = helper.getAllApplicableServices(stackInfo); - assertEquals(12, allServices.size()); + assertEquals(13, allServices.size()); for (ServiceInfo serviceInfo : allServices) { if (serviceInfo.getName().equals("NAGIOS")) { assertTrue(serviceInfo.isMonitoringService()); http://git-wip-us.apache.org/repos/asf/ambari/blob/abd9fcac/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java index c9c8258..db96059 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java @@ -7113,7 +7113,7 @@ public class AmbariManagementControllerTest { public void testGetStackServices() throws Exception { StackServiceRequest request = new StackServiceRequest(STACK_NAME, NEW_STACK_VERSION, null); Set<StackServiceResponse> responses = controller.getStackServices(Collections.singleton(request)); - Assert.assertEquals(11, responses.size()); + Assert.assertEquals(12, responses.size()); StackServiceRequest requestWithParams = new StackServiceRequest(STACK_NAME, NEW_STACK_VERSION, SERVICE_NAME); http://git-wip-us.apache.org/repos/asf/ambari/blob/abd9fcac/ambari-server/src/test/java/org/apache/ambari/server/state/ConfigHelperTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/state/ConfigHelperTest.java b/ambari-server/src/test/java/org/apache/ambari/server/state/ConfigHelperTest.java index 73004ba..1c01ec5 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/state/ConfigHelperTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/state/ConfigHelperTest.java @@ -17,6 +17,11 @@ */ package org.apache.ambari.server.state; +import static org.easymock.EasyMock.createNiceMock; +import static org.easymock.EasyMock.expect; +import static org.easymock.EasyMock.replay; +import static org.easymock.EasyMock.verify; + import com.google.inject.Guice; import com.google.inject.Injector; import com.google.inject.persist.PersistService; @@ -35,12 +40,14 @@ import org.junit.After; import org.junit.Before; import org.junit.Test; -import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; -import java.util.List; import java.util.Map; +import java.util.ArrayList; +import java.util.List; + + public class ConfigHelperTest { private Clusters clusters; @@ -101,6 +108,22 @@ public class ConfigHelperTest { managementController.updateClusters(new HashSet<ClusterRequest>() {{ add(clusterRequest1); }}, null); + // flume-conf + + ConfigurationRequest cr2 = new ConfigurationRequest(); + cr2.setClusterName(clusterName); + cr2.setType("flume-conf"); + cr2.setVersionTag("version1"); + + + final ClusterRequest clusterRequest2 = + new ClusterRequest(cluster.getClusterId(), clusterName, + cluster.getDesiredStackVersion().getStackVersion(), null); + + clusterRequest2.setDesiredConfig(Collections.singletonList(cr2)); + managementController.updateClusters(new HashSet<ClusterRequest>() + {{ add(clusterRequest2); }}, null); + // global cr.setType("global"); cr.setVersionTag("version1"); @@ -115,13 +138,13 @@ public class ConfigHelperTest { put("attribute2", attrs); }}); - final ClusterRequest clusterRequest2 = + final ClusterRequest clusterRequest3 = new ClusterRequest(cluster.getClusterId(), clusterName, cluster.getDesiredStackVersion().getStackVersion(), null); - clusterRequest2.setDesiredConfig(Collections.singletonList(cr)); + clusterRequest3.setDesiredConfig(Collections.singletonList(cr)); managementController.updateClusters(new HashSet<ClusterRequest>() - {{ add(clusterRequest2); }}, null); + {{ add(clusterRequest3); }}, null); } @After @@ -248,10 +271,10 @@ public class ConfigHelperTest { Map<String, Map<String, Map<String, String>>> effectiveAttributes = configHelper .getEffectiveConfigAttributes(cluster, - configHelper.getEffectiveDesiredTags(cluster, "h1")); + configHelper.getEffectiveDesiredTags(cluster, "h1")); Assert.assertNotNull(effectiveAttributes); - Assert.assertEquals(2, effectiveAttributes.size()); + Assert.assertEquals(3, effectiveAttributes.size()); Assert.assertTrue(effectiveAttributes.containsKey("global")); Map<String, Map<String, String>> globalAttrs = effectiveAttributes.get("global"); @@ -494,4 +517,40 @@ public class ConfigHelperTest { Assert.assertEquals("true", finalResultAttributes.get("c")); Assert.assertEquals("true", finalResultAttributes.get("d")); } + + @Test + public void testCalculateIsStaleConfigs() throws Exception { + + Map<String, HostConfig> schReturn = new HashMap<String, HostConfig>(); + HostConfig hc = new HostConfig(); + // Put a different version to check for change + hc.setDefaultVersionTag("version2"); + schReturn.put("flume-conf", hc); + // set up mocks + ServiceComponentHost sch = createNiceMock(ServiceComponentHost.class); + // set up expectations + expect(sch.getActualConfigs()).andReturn(schReturn).times(3); + expect(sch.getHostName()).andReturn("h1").times(6); + expect(sch.getClusterId()).andReturn(1l).times(3); + expect(sch.getServiceName()).andReturn("FLUME").times(3); + expect(sch.getServiceComponentName()).andReturn("FLUME_HANDLER").times(3); + replay(sch); + // Cluster level config changes + Assert.assertTrue(configHelper.isStaleConfigs(sch)); + HostConfig hc2 = new HostConfig(); + hc2.setDefaultVersionTag("version1"); + schReturn.put("flume-conf", hc2); + // invalidate cache to test new sch + configHelper.invalidateStaleConfigsCache(); + // Cluster level same configs + Assert.assertFalse(configHelper.isStaleConfigs(sch)); + // Cluster level same configs but group specific configs for host have been updated + List<String> hosts = new ArrayList<String>(); + hosts.add("h1"); + List<Config> configs = new ArrayList<Config>(); + configs.add(new ConfigImpl("flume-conf")); + addConfigGroup("configGroup1", "FLUME", hosts, configs); + Assert.assertTrue(configHelper.isStaleConfigs(sch)); + verify(sch); + } } http://git-wip-us.apache.org/repos/asf/ambari/blob/abd9fcac/ambari-server/src/test/resources/stacks/HDP/2.0.6/services/FLUME/configuration/flume-conf.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/resources/stacks/HDP/2.0.6/services/FLUME/configuration/flume-conf.xml b/ambari-server/src/test/resources/stacks/HDP/2.0.6/services/FLUME/configuration/flume-conf.xml new file mode 100644 index 0000000..74a4c15 --- /dev/null +++ b/ambari-server/src/test/resources/stacks/HDP/2.0.6/services/FLUME/configuration/flume-conf.xml @@ -0,0 +1,31 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> + +<configuration supports_final="false"> + <property> + <name>content</name> + <description>Describe all the Flume agent configurations</description> + <value> +# Flume agent config + </value> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/abd9fcac/ambari-server/src/test/resources/stacks/HDP/2.0.6/services/FLUME/configuration/flume-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/resources/stacks/HDP/2.0.6/services/FLUME/configuration/flume-env.xml b/ambari-server/src/test/resources/stacks/HDP/2.0.6/services/FLUME/configuration/flume-env.xml new file mode 100644 index 0000000..7b11bde --- /dev/null +++ b/ambari-server/src/test/resources/stacks/HDP/2.0.6/services/FLUME/configuration/flume-env.xml @@ -0,0 +1,40 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> + +<configuration> + <property> + <name>flume_conf_dir</name> + <value>/etc/flume/conf</value> + <description>Location to save configuration files</description> + </property> + <property> + <name>flume_log_dir</name> + <value>/var/log/flume</value> + <description>Location to save log files</description> + </property> + <property> + <name>flume_user</name> + <value>flume</value> + <property-type>USER</property-type> + <description>Flume User</description> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/abd9fcac/ambari-server/src/test/resources/stacks/HDP/2.0.6/services/FLUME/configuration/flume-log4j.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/resources/stacks/HDP/2.0.6/services/FLUME/configuration/flume-log4j.xml b/ambari-server/src/test/resources/stacks/HDP/2.0.6/services/FLUME/configuration/flume-log4j.xml new file mode 100644 index 0000000..8c6ac27 --- /dev/null +++ b/ambari-server/src/test/resources/stacks/HDP/2.0.6/services/FLUME/configuration/flume-log4j.xml @@ -0,0 +1,31 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> + +<configuration supports_final="false"> + <property> + <name>content</name> + <description>Custom log4j.properties</description> + <value> +# Flume log4j config + </value> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/abd9fcac/ambari-server/src/test/resources/stacks/HDP/2.0.6/services/FLUME/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/resources/stacks/HDP/2.0.6/services/FLUME/metainfo.xml b/ambari-server/src/test/resources/stacks/HDP/2.0.6/services/FLUME/metainfo.xml new file mode 100644 index 0000000..4f6bbd9 --- /dev/null +++ b/ambari-server/src/test/resources/stacks/HDP/2.0.6/services/FLUME/metainfo.xml @@ -0,0 +1,69 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<metainfo> + <schemaVersion>2.0</schemaVersion> + <services> + <service> + <name>FLUME</name> + <displayName>Flume</displayName> + <comment>Data management and processing platform</comment> + <version>1.4.0.2.0</version> + <components> + <component> + <name>FLUME_HANDLER</name> + <displayName>Flume</displayName> + <category>SLAVE</category> + <cardinality>1+</cardinality> + <commandScript> + <script>scripts/flume_handler.py</script> + <scriptType>PYTHON</scriptType> + <timeout>600</timeout> + </commandScript> + </component> + </components> + + <osSpecifics> + <osSpecific> + <osFamily>any</osFamily> + <packages> + <package> + <name>flume</name> + </package> + </packages> + </osSpecific> + </osSpecifics> + + <commandScript> + <script>scripts/flume_check.py</script> + <scriptType>PYTHON</scriptType> + <timeout>300</timeout> + </commandScript> + + <requiredServices> + <service>HDFS</service> + </requiredServices> + + <configuration-dependencies> + <config-type>flume-env</config-type> + <config-type>flume-conf</config-type> + <config-type>flume-log4j</config-type> + </configuration-dependencies> + + </service> + </services> +</metainfo>
