Updated Branches: refs/heads/trunk 1f26f6d3d -> 6115a5720
AMBARI-3600, AMBARI-3601. Add stale_configs to host_component response and config_types to stack service response (ncole) Project: http://git-wip-us.apache.org/repos/asf/incubator-ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-ambari/commit/6115a572 Tree: http://git-wip-us.apache.org/repos/asf/incubator-ambari/tree/6115a572 Diff: http://git-wip-us.apache.org/repos/asf/incubator-ambari/diff/6115a572 Branch: refs/heads/trunk Commit: 6115a5720ed206ba54fee233dd5d006c54267fde Parents: 1f26f6d Author: Nate Cole <[email protected]> Authored: Mon Oct 28 11:56:39 2013 -0400 Committer: Nate Cole <[email protected]> Committed: Tue Oct 29 12:24:45 2013 -0400 ---------------------------------------------------------------------- .../server/api/util/StackExtensionHelper.java | 6 +- .../AmbariManagementControllerImpl.java | 138 +++----- .../ServiceComponentHostResponse.java | 16 + .../server/controller/StackServiceResponse.java | 12 +- .../internal/HostComponentResourceProvider.java | 4 + .../internal/StackServiceResourceProvider.java | 8 +- .../ambari/server/state/ConfigHelper.java | 330 +++++++++++++++++++ .../apache/ambari/server/state/ServiceInfo.java | 82 ++++- .../svccomphost/ServiceComponentHostImpl.java | 20 +- .../src/main/resources/properties.json | 2 + .../AmbariManagementControllerTest.java | 3 +- .../svccomphost/ServiceComponentHostTest.java | 163 ++++++++- 12 files changed, 673 insertions(+), 111 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/6115a572/ambari-server/src/main/java/org/apache/ambari/server/api/util/StackExtensionHelper.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/util/StackExtensionHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/api/util/StackExtensionHelper.java index 8feca20..6463150 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/api/util/StackExtensionHelper.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/api/util/StackExtensionHelper.java @@ -154,7 +154,7 @@ public class StackExtensionHelper { List<ServiceInfo> serviceInfoList = parentStack.getServices(); for (ServiceInfo service : serviceInfoList) { ServiceInfo existingService = serviceInfoMap.get(service.getName()); - if (service.isDeleted().booleanValue()) { + if (service.isDeleted()) { serviceInfoMap.remove(service.getName()); continue; } @@ -185,6 +185,9 @@ public class StackExtensionHelper { .FILENAME_FILTER); if (servicesFolders != null) { for (File serviceFolder : servicesFolders) { + if (!serviceFolder.isDirectory()) + continue; + // Get information about service ServiceInfo serviceInfo = new ServiceInfo(); serviceInfo.setName(serviceFolder.getName()); @@ -314,6 +317,7 @@ public class StackExtensionHelper { pi.setFilename(propertyFile.getName()); list.add(pi); } + return list; } catch (Exception e) { LOG.error("Could not load configuration for " + propertyFile, e); http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/6115a572/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java index cd7e957..78c3c90 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java @@ -18,11 +18,21 @@ package org.apache.ambari.server.controller; -import com.google.gson.Gson; -import com.google.inject.Inject; -import com.google.inject.Injector; -import com.google.inject.Singleton; -import com.google.inject.persist.Transactional; +import java.io.File; +import java.io.IOException; +import java.net.InetAddress; +import java.text.MessageFormat; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; +import java.util.TreeMap; + import org.apache.ambari.server.AmbariException; import org.apache.ambari.server.ClusterNotFoundException; import org.apache.ambari.server.DuplicateResourceException; @@ -55,6 +65,7 @@ import org.apache.ambari.server.state.Clusters; import org.apache.ambari.server.state.ComponentInfo; import org.apache.ambari.server.state.Config; import org.apache.ambari.server.state.ConfigFactory; +import org.apache.ambari.server.state.ConfigHelper; import org.apache.ambari.server.state.DesiredConfig; import org.apache.ambari.server.state.Host; import org.apache.ambari.server.state.HostState; @@ -87,20 +98,11 @@ import org.apache.http.client.utils.URIBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.File; -import java.io.IOException; -import java.net.InetAddress; -import java.text.MessageFormat; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.Set; -import java.util.TreeMap; +import com.google.gson.Gson; +import com.google.inject.Inject; +import com.google.inject.Injector; +import com.google.inject.Singleton; +import com.google.inject.persist.Transactional; @Singleton public class AmbariManagementControllerImpl implements @@ -1247,54 +1249,15 @@ public class AmbariManagementControllerImpl implements } } - private void findConfigurationPropertiesWithOverrides( + private Map<String, Map<String, String>> findConfigurationPropertiesWithOverrides( Map<String, Map<String, String>> configurations, - Map<String, Map<String, String>> configTags, Cluster cluster, String serviceName, - Map<String, DesiredConfig> clusterDesiredConfigs, Map<String, DesiredConfig> desiredConfigMap) throws AmbariException { - // Do not use host component config mappings. Instead, the rules are: - // 1) Use the cluster desired config - // 2) override (1) with service-specific overrides - // 3) override (2) with host-specific overrides - - for (Entry<String, DesiredConfig> entry : clusterDesiredConfigs.entrySet()) { - String type = entry.getKey(); - String tag = entry.getValue().getVersion(); - // 1) start with cluster config - Config config = cluster.getConfig(type, tag); - - if (null == config) { - continue; - } - - Map<String, String> props = new HashMap<String, String>(config.getProperties()); - Map<String, String> tags = new HashMap<String, String>(); - tags.put(CLUSTER_LEVEL_TAG, config.getVersionTag()); - - // 2) apply the service overrides, if any are defined with different tags - Service service = cluster.getService(serviceName); - Config svcConfig = service.getDesiredConfigs().get(type); - if (null != svcConfig && !svcConfig.getVersionTag().equals(tag)) { - props.putAll(svcConfig.getProperties()); - //TODO why don't update tags with service overrides? - tags.put("service_override_tag", svcConfig.getVersionTag()); - } - - // 3) apply the host overrides, if any - DesiredConfig dc = desiredConfigMap.get(type); - - if (null != dc) { - Config hostConfig = cluster.getConfig(type, dc.getVersion()); - if (null != hostConfig) { - props.putAll(hostConfig.getProperties()); - tags.put("host_override_tag", hostConfig.getVersionTag()); - } - } - - configTags.put(type, tags); - } + + ConfigHelper ch = injector.getInstance(ConfigHelper.class); + + Map<String, Map<String,String>> configTags = ch.getEffectiveDesiredTags(cluster, serviceName, desiredConfigMap); // HACK HACK HACK if the service has configs that are NOT included @@ -1313,32 +1276,21 @@ public class AmbariManagementControllerImpl implements configTags.put(type, tags); } } + + return configTags; } - private void findConfigurationPropertiesWithOverrides( + private Map<String, Map<String,String>> findConfigurationPropertiesWithOverrides( Map<String, Map<String, String>> configurations, - Map<String, Map<String, String>> configTags, - Cluster cluster, String serviceName, String hostName, - Map<String, DesiredConfig> clusterDesiredConfigs) throws AmbariException { + Cluster cluster, String serviceName, String hostName) throws AmbariException { Host host = clusters.getHost(hostName); Map<String, DesiredConfig> desiredConfigMap = host.getDesiredConfigs(cluster.getClusterId()); - findConfigurationPropertiesWithOverrides(configurations, configTags, cluster, - serviceName, clusterDesiredConfigs, desiredConfigMap); + return findConfigurationPropertiesWithOverrides(configurations, cluster, + serviceName, desiredConfigMap); } - private void findConfigurationPropertiesWithOverrides( - Map<String, Map<String, String>> configurations, - Map<String, Map<String, String>> configTags, - Cluster cluster, String serviceName, String hostName) throws AmbariException { - - Map<String, DesiredConfig> clusterDesiredConfigs = cluster.getDesiredConfigs(); - - findConfigurationPropertiesWithOverrides(configurations, configTags, cluster, - serviceName, hostName, clusterDesiredConfigs); - - } private List<Stage> doStageCreation(Cluster cluster, Map<State, List<Service>> changedServices, @@ -1348,7 +1300,6 @@ public class AmbariManagementControllerImpl implements boolean runSmokeTest, boolean reconfigureClients) throws AmbariException { - Map<String, DesiredConfig> clusterDesiredConfigs = cluster.getDesiredConfigs(); // TODO handle different transitions? // Say HDFS to stopped and MR to started, what order should actions be done @@ -1531,10 +1482,8 @@ public class AmbariManagementControllerImpl implements // [ type -> [ key, value ] ] Map<String, Map<String, String>> configurations = new TreeMap<String, Map<String, String>>(); - Map<String, Map<String, String>> configTags = new HashMap<String, Map<String, String>>(); - - findConfigurationPropertiesWithOverrides(configurations, configTags, cluster, scHost.getServiceName(), - clusterDesiredConfigs, configsByHosts.get(scHost.getHostName())); + Map<String, Map<String, String>> configTags = findConfigurationPropertiesWithOverrides( + configurations, cluster, scHost.getServiceName(), configsByHosts.get(scHost.getHostName())); // HACK HACK HACK if (!scHost.getHostName().equals(jobtrackerHost)) { @@ -1577,10 +1526,8 @@ public class AmbariManagementControllerImpl implements // [ type -> [ key, value ] ] Map<String, Map<String, String>> configurations = new TreeMap<String, Map<String,String>>(); - Map<String, Map<String, String>> configTags = new HashMap<String, Map<String,String>>(); - - findConfigurationPropertiesWithOverrides(configurations, configTags, - cluster, serviceName, clientHost, clusterDesiredConfigs); + Map<String, Map<String, String>> configTags = findConfigurationPropertiesWithOverrides( + configurations, cluster, serviceName, clientHost); stage.getExecutionCommandWrapper(clientHost, smokeTestRole).getExecutionCommand() @@ -2739,10 +2686,7 @@ public class AmbariManagementControllerImpl implements // [ type -> [ key, value ] ] Map<String, Map<String, String>> configurations = new TreeMap<String, Map<String,String>>(); - Map<String, Map<String, String>> configTags = new TreeMap<String, - Map<String, String>>(); - - findConfigurationPropertiesWithOverrides(configurations, configTags, + Map<String, Map<String, String>> configTags = findConfigurationPropertiesWithOverrides(configurations, cluster, actionRequest.getServiceName(), hostName); ExecutionCommand execCmd = stage.getExecutionCommandWrapper(hostName, @@ -2793,11 +2737,11 @@ public class AmbariManagementControllerImpl implements Map<String, Map<String, String>> configurations = new TreeMap<String, Map<String, String>>(); - Map<String, Map<String, String>> configTags = new TreeMap<String, - Map<String, String>>(); - findConfigurationPropertiesWithOverrides(configurations, configTags, - cluster, serviceName, namenodeHost); + + Map<String, Map<String, String>> configTags = findConfigurationPropertiesWithOverrides( + configurations, cluster, serviceName, namenodeHost); + // Add the tag for hdfs-exclude-file Map<String, String> excludeTags = new HashMap<String, String>(); excludeTags.put(CLUSTER_LEVEL_TAG, config.getVersionTag()); http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/6115a572/ambari-server/src/main/java/org/apache/ambari/server/controller/ServiceComponentHostResponse.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/ServiceComponentHostResponse.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/ServiceComponentHostResponse.java index b3fdbee..eba7da8 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/ServiceComponentHostResponse.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/ServiceComponentHostResponse.java @@ -50,6 +50,8 @@ public class ServiceComponentHostResponse { private String ha_status = "NA"; + private boolean staleConfig = false; + public ServiceComponentHostResponse(String clusterName, String serviceName, String componentName, String hostname, @@ -271,4 +273,18 @@ public class ServiceComponentHostResponse { return actualConfigs; } + /** + * @return if the configs are stale + */ + public boolean isStaleConfig() { + return staleConfig; + } + + /** + * @param stale + */ + public void setStaleConfig(boolean stale) { + staleConfig = stale; + } + } http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/6115a572/ambari-server/src/main/java/org/apache/ambari/server/controller/StackServiceResponse.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/StackServiceResponse.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/StackServiceResponse.java index d9cbf6e..4aa3f49 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/StackServiceResponse.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/StackServiceResponse.java @@ -18,6 +18,8 @@ package org.apache.ambari.server.controller; +import java.util.List; + public class StackServiceResponse { private String serviceName; @@ -27,12 +29,16 @@ public class StackServiceResponse { private String comments; private String serviceVersion; + + private List<String> configTypes; - public StackServiceResponse(String serviceName, String userName, String comments, String serviceVersion) { + public StackServiceResponse(String serviceName, String userName, String comments, String serviceVersion, + List<String> types) { setServiceName(serviceName); setUserName(userName); setComments(comments); setServiceVersion(serviceVersion); + configTypes = types; } public String getServiceName() { @@ -66,5 +72,9 @@ public class StackServiceResponse { public void setServiceVersion(String serviceVersion) { this.serviceVersion = serviceVersion; } + + public List<String> getConfigTypes() { + return configTypes; + } } http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/6115a572/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/HostComponentResourceProvider.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/HostComponentResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/HostComponentResourceProvider.java index f9dd801..006e59b 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/HostComponentResourceProvider.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/HostComponentResourceProvider.java @@ -66,6 +66,8 @@ class HostComponentResourceProvider extends AbstractControllerResourceProvider { = PropertyHelper.getPropertyId("HostRoles", "desired_stack_id"); protected static final String HOST_COMPONENT_ACTUAL_CONFIGS_PROPERTY_ID = PropertyHelper.getPropertyId("HostRoles", "actual_configs"); + protected static final String HOST_COMPONENT_STALE_CONFIGS_PROPERTY_ID + = PropertyHelper.getPropertyId("HostRoles", "stale_configs"); //Component name mappings private static final Map<String, PropertyProvider> HOST_COMPONENT_PROPERTIES_PROVIDER = new HashMap<String, PropertyProvider>(); @@ -186,6 +188,8 @@ class HostComponentResourceProvider extends AbstractControllerResourceProvider { response.getDesiredConfigs(), requestedIds); setResourceProperty(resource, HOST_COMPONENT_ACTUAL_CONFIGS_PROPERTY_ID, response.getActualConfigs(), requestedIds); + setResourceProperty(resource, HOST_COMPONENT_STALE_CONFIGS_PROPERTY_ID, + Boolean.valueOf(response.isStaleConfig()), requestedIds); String componentName = (String)resource.getPropertyValue(HOST_COMPONENT_COMPONENT_NAME_PROPERTY_ID); PropertyProvider propertyProvider = HOST_COMPONENT_PROPERTIES_PROVIDER.get(componentName); http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/6115a572/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackServiceResourceProvider.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackServiceResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackServiceResourceProvider.java index 1c45537..3265782 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackServiceResourceProvider.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackServiceResourceProvider.java @@ -58,7 +58,10 @@ public class StackServiceResourceProvider extends ReadOnlyResourceProvider { .getPropertyId("StackServices", "comments"); private static final String VERSION_PROPERTY_ID = PropertyHelper - .getPropertyId("StackServices", "service_version");; + .getPropertyId("StackServices", "service_version"); + + private static final String CONFIG_TYPES = PropertyHelper + .getPropertyId("StackServices", "config_types"); private static Set<String> pkPropertyIds = new HashSet<String>( Arrays.asList(new String[] { STACK_NAME_PROPERTY_ID, @@ -109,6 +112,9 @@ public class StackServiceResourceProvider extends ReadOnlyResourceProvider { setResourceProperty(resource, VERSION_PROPERTY_ID, response.getServiceVersion(), requestedIds); + + setResourceProperty(resource, CONFIG_TYPES, + response.getConfigTypes(), requestedIds); resources.add(resource); } http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/6115a572/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java new file mode 100644 index 0000000..855711b --- /dev/null +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java @@ -0,0 +1,330 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ambari.server.state; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; +import java.util.TreeMap; + +import org.apache.ambari.server.AmbariException; +import org.apache.ambari.server.api.services.AmbariMetaInfo; + +import com.google.inject.Inject; + +/** + * Helper class that works with config traversals. + */ +public class ConfigHelper { + + private Clusters clusters = null; + private AmbariMetaInfo ambariMetaInfo = null; + + @Inject + public ConfigHelper(Clusters c, AmbariMetaInfo metaInfo) { + clusters = c; + ambariMetaInfo = metaInfo; + } + + /** + * Gets the desired tags for a cluster and host + * @param cluster the cluster + * @param serviceName the optional service name + * @param hostName the host name + * @return a map of tag type to tag names with overrides + * @throws AmbariException + */ + public Map<String, Map<String, String>> getEffectiveDesiredTags( + Cluster cluster, String serviceName, String hostName) throws AmbariException { + + Host host = clusters.getHost(hostName); + Map<String, DesiredConfig> hostDesired = host.getDesiredConfigs(cluster.getClusterId()); + + return getEffectiveDesiredTags(cluster, serviceName, hostDesired); + } + + /** + * Gets the desired tags for a cluster and host + * @param cluster the cluster + * @param serviceName the optional service name + * @param hostDesired the optional host desired configs + * @return a map of tag type to tag names with overrides + * @throws AmbariException + */ + public Map<String, Map<String, String>> getEffectiveDesiredTags( + Cluster cluster, String serviceName, Map<String, DesiredConfig> hostDesired) throws AmbariException { + + Map<String, DesiredConfig> clusterDesired = cluster.getDesiredConfigs(); + + Map<String, Map<String,String>> resolved = new TreeMap<String, Map<String, String>>(); + + // Do not use host component config mappings. Instead, the rules are: + // 1) Use the cluster desired config + // 2) override (1) with service-specific overrides + // 3) override (2) with host-specific overrides + + for (Entry<String, DesiredConfig> clusterEntry : clusterDesired.entrySet()) { + String type = clusterEntry.getKey(); + String tag = clusterEntry.getValue().getVersion(); + + // 1) start with cluster config + Config config = cluster.getConfig(type, tag); + if (null == config) { + continue; + } + + Map<String, String> tags = new LinkedHashMap<String, String>(); + + tags.put("tag", config.getVersionTag()); + + // 2) apply the service overrides, if any are defined with different tags + if (null != serviceName) { + Service service = cluster.getService(serviceName); + Config svcConfig = service.getDesiredConfigs().get(type); + if (null != svcConfig && !svcConfig.getVersionTag().equals(tag)) { + tags.put("service_override_tag", svcConfig.getVersionTag()); + } + } + + if (null != hostDesired) { + // 3) apply the host overrides, if any + DesiredConfig dc = hostDesired.get(type); + + if (null != dc) { + Config hostConfig = cluster.getConfig(type, dc.getVersion()); + if (null != hostConfig) { + tags.put("host_override_tag", hostConfig.getVersionTag()); + } + } + } + + resolved.put(type, tags); + } + + return resolved; + } + + /** + * The purpose of this method is to determine if a {@link ServiceComponentHost}'s + * known actual configs are different than what is set on the cluster (the desired). + * The following logic is applied: + * <ul> + * <li>Desired type does not exist on the SCH (actual) + * <ul> + * <li>Type does not exist on the stack: <code>false</code></li> + * <li>Type exists on the stack: <code>true</code> if the config key is on the stack. + * otherwise <code>false</code></li> + * </ul> + * </li> + * <li> Desired type exists for the SCH + * <ul> + * <li>Desired tags already set for the SCH (actual): <code>false</code></li> + * <li>Desired tags DO NOT match SCH: <code>true</code> if the changed keys + * exist on the stack, otherwise <code>false</code></li> + * </ul> + * </li> + * </ul> + * @param serviceComponentHostImpl + * @return <code>true</code> if the actual configs are stale + */ + public boolean isStaleConfigs(ServiceComponentHost sch) throws AmbariException { + + Map<String, DesiredConfig> actual = sch.getActualConfigs(); + if (null == actual || actual.isEmpty()) + return false; + + Cluster cluster = clusters.getClusterById(sch.getClusterId()); + StackId stackId = cluster.getDesiredStackVersion(); + + Map<String, Map<String, String>> desired = getEffectiveDesiredTags(cluster, + sch.getServiceName(), sch.getHostName()); + + ServiceInfo serviceInfo = ambariMetaInfo.getService(stackId.getStackName(), + stackId.getStackVersion(), sch.getServiceName()); + + // Configs are considered stale when: + // - desired type DOES NOT exist in actual + // --- desired type DOES NOT exist in stack: not_stale + // --- desired type DOES exist in stack: check stack for any key: stale + // - desired type DOES exist in actual + // --- desired tags DO match actual tags: not_stale + // --- desired tags DO NOT match actual tags + // ---- merge values, determine changed keys, check stack: stale + + boolean stale = false; + + Iterator<Entry<String, Map<String, String>>> it = desired.entrySet().iterator(); + + while (it.hasNext() && !stale) { + Entry<String, Map<String, String>> desiredEntry = it.next(); + + String type = desiredEntry.getKey(); + Map<String, String> tags = desiredEntry.getValue(); + + if (!actual.containsKey(type)) { + // desired is set, but actual is not + if (!serviceInfo.hasConfigType(type)) { + stale = false; + } else { + // find out if the keys are stale by first checking the target service, + // then all services + Collection<String> keys = mergeKeyNames(cluster, type, tags.values()); + + if (serviceInfo.hasPropertyFor(type, keys) || !hasPropertyFor(stackId, type, keys)) + stale = true; + } + } else { + // desired and actual both define the type + DesiredConfig dc = actual.get(type); + Map<String, String> actualTags = buildTags(dc); + + if (!isTagChange(tags, actualTags)) { + stale = false; + } else { + // tags are change, need to find out what has changed, and if it applies + // to the service + Collection<String> changed = findChangedKeys(cluster, type, tags.values(), actualTags.values()); + if (serviceInfo.hasPropertyFor(type, changed)) { + stale = true; + } + } + + } + } + return stale; + } + + /** + * @return <code>true</code> if any service on the stack defines a property + * for the type. + */ + private boolean hasPropertyFor(StackId stack, String type, + Collection<String> keys) throws AmbariException { + + for (ServiceInfo svc : ambariMetaInfo.getServices(stack.getStackName(), + stack.getStackVersion()).values()) { + + if (svc.hasPropertyFor(type, keys)) + return true; + + } + + return false; + } + + /** + * @return the keys that have changed values + */ + private Collection<String> findChangedKeys(Cluster cluster, String type, + Collection<String> desiredTags, Collection<String> actualTags) { + + Map<String, String> desiredValues = new HashMap<String, String>(); + Map<String, String> actualValues = new HashMap<String, String>(); + + for (String tag : desiredTags) { + Config config = cluster.getConfig(type, tag); + if (null != config) + desiredValues.putAll(config.getProperties()); + } + + for (String tag : actualTags) { + Config config = cluster.getConfig(type, tag); + if (null != config) + actualValues.putAll(config.getProperties()); + } + + List<String> keys = new ArrayList<String>(); + + for (Entry<String, String> entry : desiredValues.entrySet()) { + String key = entry.getKey(); + String value = entry.getValue(); + + if (!actualValues.containsKey(key)) + keys.add(key); + else if (!actualValues.get(key).equals(value)) + keys.add(key); + } + + return keys; + } + + /** + * @return the map of tags for a desired config + */ + private Map<String, String> buildTags(DesiredConfig dc) { + Map<String, String> map = new LinkedHashMap<String, String>(); + map.put("tag", dc.getVersion()); + if (null != dc.getServiceName()) + map.put("service_override_tag", dc.getServiceName()); + if (0 != dc.getHostOverrides().size()) + map.put("host_override_tag", dc.getHostOverrides().get(0).getVersionTag()); + + return map; + } + + /** + * @return true if the tags are different in any way, even if not-specified + */ + private boolean isTagChange(Map<String, String> desiredTags, Map<String, String> actualTags) { + if (!actualTags.get("tag").equals (desiredTags.get("tag"))) + return true; + + String tag0 = actualTags.get("service_override_tag"); + String tag1 = desiredTags.get("service_override_tag"); + tag0 = (null == tag0) ? "" : tag0; + tag1 = (null == tag1) ? "" : tag1; + if (!tag0.equals(tag1)) + return true; + + // desired config can only have one value here since it's from the HC. + tag0 = actualTags.get("host_override_tag"); + tag1 = desiredTags.get("host_override_tag"); + tag0 = (null == tag0) ? "" : tag0; + tag1 = (null == tag1) ? "" : tag1; + if (!tag0.equals(tag1)) + return true; + + return false; + } + + /** + * @return the list of combined config property names + */ + private Collection<String> mergeKeyNames(Cluster cluster, String type, Collection<String> tags) { + Set<String> names = new HashSet<String>(); + + for (String tag : tags) { + Config config = cluster.getConfig(type, tag); + if (null != config) { + names.addAll(config.getProperties().keySet()); + } + } + + return names; + } + + + +} http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/6115a572/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceInfo.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceInfo.java b/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceInfo.java index 937d1de..fa2c759 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceInfo.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceInfo.java @@ -19,9 +19,15 @@ package org.apache.ambari.server.state; import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; import java.util.List; +import java.util.Map; +import java.util.Set; import org.apache.ambari.server.controller.StackServiceResponse; +import org.codehaus.jackson.annotate.JsonIgnore; import org.codehaus.jackson.map.annotate.JsonFilter; @JsonFilter("propertiesfilter") @@ -32,14 +38,16 @@ public class ServiceInfo { private String comment; private List<PropertyInfo> properties; private List<ComponentInfo> components; - private Boolean isDeleted = false; + private boolean isDeleted = false; + @JsonIgnore + private volatile Map<String, Set<String>> configLayout = null; - public Boolean isDeleted() { + public boolean isDeleted() { return isDeleted; } public void setDeleted(boolean deleted) { - isDeleted = Boolean.valueOf(deleted); + isDeleted = deleted; } public String getName() { @@ -128,6 +136,72 @@ public class ServiceInfo { public StackServiceResponse convertToResponse() { - return new StackServiceResponse(getName(), getUser(), getComment(), getVersion()); + return new StackServiceResponse(getName(), getUser(), getComment(), getVersion(), + getConfigTypes()); + } + + public List<String> getConfigTypes() { + buildConfigLayout(); + return new ArrayList<String>(configLayout.keySet()); + } + + + /** + * @param type the config type + * @return <code>true</code> if the service defines the supplied type + */ + public boolean hasConfigType(String type) { + buildConfigLayout(); + + return configLayout.containsKey(type); + } + + /** + * The purpose of this method is to determine if a service has a property + * defined in a supplied set: + * <ul> + * <li>If the type is not defined for the service, then no property can exist.</li> + * <li>If the type is defined, then check each supplied property for existence.</li> + * </ul> + * @param type the config type + * @param keyNames the names of all the config keys for the given type + * @return <code>true</code> if the config is stale + */ + public boolean hasPropertyFor(String type, Collection<String> keyNames) { + if (!hasConfigType(type)) + return false; + + Set<String> keys = configLayout.get(type); + + for (String staleCheck : keyNames) { + if (keys.contains(staleCheck)) + return true; + } + + return false; + } + + /** + * Builds the config map specific to this service. + */ + private void buildConfigLayout() { + if (null == configLayout) { + synchronized(this) { + if (null == configLayout) { + configLayout = new HashMap<String, Set<String>>(); + + for (PropertyInfo pi : getProperties()) { + String type = pi.getFilename(); + int idx = type.indexOf(".xml"); + type = type.substring(0, idx); + + if (!configLayout.containsKey(type)) + configLayout.put(type, new HashSet<String>()); + + configLayout.get(type).add(pi.getName()); + } + } + } + } } } http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/6115a572/ambari-server/src/main/java/org/apache/ambari/server/state/svccomphost/ServiceComponentHostImpl.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/svccomphost/ServiceComponentHostImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/state/svccomphost/ServiceComponentHostImpl.java index 1e49808..2c79f96 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/svccomphost/ServiceComponentHostImpl.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/svccomphost/ServiceComponentHostImpl.java @@ -87,11 +87,12 @@ public class ServiceComponentHostImpl implements ServiceComponentHost { @Inject Clusters clusters; @Inject - HostComponentDesiredConfigMappingDAO - hostComponentDesiredConfigMappingDAO; + HostComponentDesiredConfigMappingDAO hostComponentDesiredConfigMappingDAO; @Inject - HostComponentConfigMappingDAO - hostComponentConfigMappingDAO; + HostComponentConfigMappingDAO hostComponentConfigMappingDAO; + + @Inject + ConfigHelper helper; private HostComponentStateEntity stateEntity; private HostComponentDesiredStateEntity desiredStateEntity; @@ -1316,6 +1317,13 @@ public class ServiceComponentHostImpl implements ServiceComponentHost { r.setHa_status(ha_status); r.setActualConfigs(actualConfigs); + + try { + r.setStaleConfig(helper.isStaleConfigs(this)); + } catch (Exception e) { + LOG.error("Could not determine stale config", e); + } + return r; } finally { readLock.unlock(); @@ -1559,6 +1567,7 @@ public class ServiceComponentHostImpl implements ServiceComponentHost { try { writeLock.lock(); try { + actualConfigs = new HashMap<String, DesiredConfig>(); String hostName = getHostName(); @@ -1571,6 +1580,7 @@ public class ServiceComponentHostImpl implements ServiceComponentHost { String hostTag = values.get("host_override_tag"); DesiredConfig dc = new DesiredConfig(); + dc.setServiceName(values.get("service_override_tag")); dc.setVersion(tag); actualConfigs.put(type, dc); if (null != hostTag && null != hostName) { @@ -1619,4 +1629,6 @@ public class ServiceComponentHostImpl implements ServiceComponentHost { } } + + } http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/6115a572/ambari-server/src/main/resources/properties.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/properties.json b/ambari-server/src/main/resources/properties.json index 00abfac..2a985c7 100644 --- a/ambari-server/src/main/resources/properties.json +++ b/ambari-server/src/main/resources/properties.json @@ -68,6 +68,7 @@ "params/run_smoke_test", "HostRoles/nagios_alerts", "HostRoles/ha_status", + "HostRoles/stale_configs", "_" ], "Configuration":[ @@ -156,6 +157,7 @@ "StackServices/user_name", "StackServices/comments", "StackServices/service_version", + "StackServices/config_types", "_" ], "StackConfiguration":[ http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/6115a572/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java index 2206062..1e8551a 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java @@ -5794,8 +5794,9 @@ public class AmbariManagementControllerTest { Assert.assertEquals(1, responsesWithParams.size()); for (StackServiceResponse responseWithParams: responsesWithParams) { Assert.assertEquals(responseWithParams.getServiceName(), SERVICE_NAME); - + Assert.assertTrue(responseWithParams.getConfigTypes().size() > 0); } + StackServiceRequest invalidRequest = new StackServiceRequest(STACK_NAME, STACK_VERSION, NON_EXT_VALUE); try { http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/6115a572/ambari-server/src/test/java/org/apache/ambari/server/state/svccomphost/ServiceComponentHostTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/state/svccomphost/ServiceComponentHostTest.java b/ambari-server/src/test/java/org/apache/ambari/server/state/svccomphost/ServiceComponentHostTest.java index b47c64f..050defa 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/state/svccomphost/ServiceComponentHostTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/state/svccomphost/ServiceComponentHostTest.java @@ -99,12 +99,21 @@ public class ServiceComponentHostTest { public void teardown() { injector.getInstance(PersistService.class).stop(); } - + private ServiceComponentHost createNewServiceComponentHost( String svc, String svcComponent, String hostName, boolean isClient) throws AmbariException{ Cluster c = clusters.getCluster("C1"); + + return createNewServiceComponentHost(c, svc, svcComponent, hostName, isClient); + } + private ServiceComponentHost createNewServiceComponentHost( + Cluster c, + String svc, + String svcComponent, + String hostName, boolean isClient) throws AmbariException{ + Service s = null; try { @@ -133,7 +142,7 @@ public class ServiceComponentHostTest { impl.getState()); Assert.assertEquals(State.INIT, impl.getDesiredState()); - Assert.assertEquals("C1", impl.getClusterName()); + Assert.assertEquals(c.getClusterName(), impl.getClusterName()); Assert.assertEquals(c.getClusterId(), impl.getClusterId()); Assert.assertEquals(s.getName(), impl.getServiceName()); Assert.assertEquals(sc.getName(), impl.getServiceComponentName()); @@ -525,6 +534,8 @@ public class ServiceComponentHostTest { Assert.assertEquals(State.INSTALLED.toString(), r.getDesiredState()); Assert.assertEquals(State.INSTALLING.toString(), r.getLiveState()); Assert.assertEquals("HDP-1.0.0", r.getStackVersion()); + + Assert.assertFalse(r.isStaleConfig()); // TODO check configs @@ -713,4 +724,152 @@ public class ServiceComponentHostTest { } } } + + @Test + public void testStaleConfigs() throws Exception { + String stackVersion="HDP-2.0.6"; + String clusterName = "c2"; + String hostName = "h3"; + + clusters.addCluster(clusterName); + clusters.addHost(hostName); + clusters.getHost(hostName).setOsType("centos5"); + clusters.getHost(hostName).persist(); + clusters.getCluster(clusterName).setDesiredStackVersion( + new StackId(stackVersion)); + metaInfo.init(); + clusters.mapHostToCluster(hostName, clusterName); + + Cluster cluster = clusters.getCluster(clusterName); + + ServiceComponentHost sch1 = createNewServiceComponentHost(cluster, "HDFS", "NAMENODE", hostName, false); + ServiceComponentHost sch2 = createNewServiceComponentHost(cluster, "HDFS", "DATANODE", hostName, false); + ServiceComponentHost sch3 = createNewServiceComponentHost(cluster, "MAPREDUCE2", "HISTORYSERVER", hostName, false); + + sch1.setDesiredState(State.INSTALLED); + sch1.setState(State.INSTALLING); + sch1.setStackVersion(new StackId(stackVersion)); + + sch2.setDesiredState(State.INSTALLED); + sch2.setState(State.INSTALLING); + sch2.setStackVersion(new StackId(stackVersion)); + + sch3.setDesiredState(State.INSTALLED); + sch3.setState(State.INSTALLING); + sch3.setStackVersion(new StackId(stackVersion)); + + Assert.assertFalse(sch1.convertToResponse().isStaleConfig()); + Assert.assertFalse(sch2.convertToResponse().isStaleConfig()); + + makeConfig(cluster, "global", "version1", + new HashMap<String,String>() {{ + put("a", "b"); + put("dfs_namenode_name_dir", "/foo1"); // HDFS only + put("mapred_log_dir_prefix", "/foo2"); // MR2 only + }}); + + Map<String, Map<String, String>> actual = new HashMap<String, Map<String, String>>() {{ + put("global", new HashMap<String,String>() {{ put("tag", "version1"); }}); + }}; + + sch1.updateActualConfigs(actual); + sch2.updateActualConfigs(actual); + sch3.updateActualConfigs(actual); + + makeConfig(cluster, "foo", "version1", + new HashMap<String,String>() {{ put("a", "c"); }}); + + // HDP-x/HDFS does not define type 'foo', so changes do not count to stale + Assert.assertFalse(sch1.convertToResponse().isStaleConfig()); + Assert.assertFalse(sch2.convertToResponse().isStaleConfig()); + + makeConfig(cluster, "hdfs-site", "version1", + new HashMap<String,String>() {{ put("a", "b"); }}); + + // HDP-x/HDFS/hdfs-site is not on the actual, but it is defined, so it is stale + Assert.assertTrue(sch1.convertToResponse().isStaleConfig()); + Assert.assertTrue(sch2.convertToResponse().isStaleConfig()); + + actual.put("hdfs-site", new HashMap<String, String>() {{ put ("tag", "version1"); }}); + + sch1.updateActualConfigs(actual); + // HDP-x/HDFS/hdfs-site up to date, only for sch1 + Assert.assertFalse(sch1.convertToResponse().isStaleConfig()); + Assert.assertTrue(sch2.convertToResponse().isStaleConfig()); + + sch2.updateActualConfigs(actual); + // HDP-x/HDFS/hdfs-site up to date for both + Assert.assertFalse(sch1.convertToResponse().isStaleConfig()); + Assert.assertFalse(sch2.convertToResponse().isStaleConfig()); + + makeConfig(cluster, "hdfs-site", "version2", + new HashMap<String, String>() {{ put("dfs.journalnode.http-address", "http://foo";); }}); + + // HDP-x/HDFS/hdfs-site updated to changed property + Assert.assertTrue(sch1.convertToResponse().isStaleConfig()); + Assert.assertTrue(sch2.convertToResponse().isStaleConfig()); + + actual.get("hdfs-site").put("tag", "version2"); + sch1.updateActualConfigs(actual); + sch2.updateActualConfigs(actual); + // HDP-x/HDFS/hdfs-site updated to changed property + Assert.assertFalse(sch1.convertToResponse().isStaleConfig()); + Assert.assertFalse(sch2.convertToResponse().isStaleConfig()); + + // make a host override + Host host = clusters.getHostsForCluster(cluster.getClusterName()).get(hostName); + Assert.assertNotNull(host); + + Config c = configFactory.createNew(cluster, "hdfs-site", + new HashMap<String, String>() {{ put("dfs.journalnode.http-address", "http://goo";); }}); + c.setVersionTag("version3"); + c.persist(); + cluster.addConfig(c); + host.addDesiredConfig(cluster.getClusterId(), true, "user", c); + + // HDP-x/HDFS/hdfs-site updated host to changed property + Assert.assertTrue(sch1.convertToResponse().isStaleConfig()); + Assert.assertTrue(sch2.convertToResponse().isStaleConfig()); + + actual.get("hdfs-site").put("host_override_tag", "version3"); + sch2.updateActualConfigs(actual); + // HDP-x/HDFS/hdfs-site updated host to changed property + Assert.assertTrue(sch1.convertToResponse().isStaleConfig()); + Assert.assertFalse(sch2.convertToResponse().isStaleConfig()); + + sch1.updateActualConfigs(actual); + // HDP-x/HDFS/hdfs-site updated host to changed property + Assert.assertFalse(sch1.convertToResponse().isStaleConfig()); + Assert.assertFalse(sch2.convertToResponse().isStaleConfig()); + + // change 'global' property only affecting global/HDFS + makeConfig(cluster, "global", "version2", + new HashMap<String,String>() {{ + put("a", "b"); + put("dfs_namenode_name_dir", "/foo3"); // HDFS only + put("mapred_log_dir_prefix", "/foo2"); // MR2 only + }}); + + Assert.assertTrue(sch1.convertToResponse().isStaleConfig()); + Assert.assertTrue(sch2.convertToResponse().isStaleConfig()); + Assert.assertFalse(sch3.convertToResponse().isStaleConfig()); + } + + /** + * Helper method to create a configuration + * @param cluster the cluster + * @param type the config type + * @param tag the config tag + * @param values the values for the config + */ + private void makeConfig(Cluster cluster, String type, String tag, Map<String, String> values) { + Config config = configFactory.createNew(cluster, type, values); + config.setVersionTag(tag); + config.persist(); + cluster.addConfig(config); + cluster.addDesiredConfig("user", config); + } + + + }
