http://git-wip-us.apache.org/repos/asf/ambari/blob/c9f0dd0b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BaseBlueprintProcessor.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BaseBlueprintProcessor.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BaseBlueprintProcessor.java deleted file mode 100644 index 73ea1a5..0000000 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BaseBlueprintProcessor.java +++ /dev/null @@ -1,771 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.ambari.server.controller.internal; - -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Map; -import java.util.Set; - -import org.apache.ambari.server.AmbariException; -import org.apache.ambari.server.StackAccessException; -import org.apache.ambari.server.api.services.AmbariMetaInfo; -import org.apache.ambari.server.controller.AmbariManagementController; -import org.apache.ambari.server.controller.spi.NoSuchParentResourceException; -import org.apache.ambari.server.controller.spi.Resource; -import org.apache.ambari.server.controller.spi.ResourceAlreadyExistsException; -import org.apache.ambari.server.controller.spi.ResourceProvider; -import org.apache.ambari.server.controller.spi.SystemException; -import org.apache.ambari.server.controller.spi.UnsupportedPropertyException; -import org.apache.ambari.server.orm.dao.BlueprintDAO; -import org.apache.ambari.server.orm.dao.StackDAO; -import org.apache.ambari.server.orm.entities.BlueprintConfigEntity; -import org.apache.ambari.server.orm.entities.BlueprintEntity; -import org.apache.ambari.server.orm.entities.HostGroupComponentEntity; -import org.apache.ambari.server.orm.entities.HostGroupConfigEntity; -import org.apache.ambari.server.orm.entities.HostGroupEntity; -import org.apache.ambari.server.state.AutoDeployInfo; -import org.apache.ambari.server.state.ConfigHelper; -import org.apache.ambari.server.state.DependencyInfo; - -import com.google.gson.Gson; - -/** - * Base blueprint processing resource provider. - */ -//todo: this class needs to be refactored to a ClusterTopology class which -//todo: has hostgroup, stack and configuration state specific to a deployment. -public abstract class BaseBlueprintProcessor extends AbstractControllerResourceProvider { - - /** - * Data access object used to obtain blueprint entities. - */ - protected static BlueprintDAO blueprintDAO; - - /** - * Data access object used to lookup value stacks parsed from the resources. - */ - protected static StackDAO stackDAO; - - /** - * Stack related information. - */ - protected static AmbariMetaInfo stackInfo; - - protected static ConfigHelper configHelper; - - - protected BaseBlueprintProcessor(Set<String> propertyIds, - Map<Resource.Type, String> keyPropertyIds, - AmbariManagementController managementController) { - - super(propertyIds, keyPropertyIds, managementController); - } - - /** - * Get host groups which contain a component. - * - * @param component component name - * @param hostGroups collection of host groups to check - * - * @return collection of host groups which contain the specified component - */ - protected Collection<HostGroupImpl> getHostGroupsForComponent(String component, Collection<HostGroupImpl> hostGroups) { - Collection<HostGroupImpl> resultGroups = new HashSet<HostGroupImpl>(); - for (HostGroupImpl group : hostGroups ) { - if (group.getComponents().contains(component)) { - resultGroups.add(group); - } - } - return resultGroups; - } - - /** - * Parse blueprint host groups. - * - * @param blueprint associated blueprint - * @param stack associated stack - * - * @return map of host group name to host group - */ - protected Map<String, HostGroupImpl> parseBlueprintHostGroups(BlueprintEntity blueprint, Stack stack) { - Map<String, HostGroupImpl> mapHostGroups = new HashMap<String, HostGroupImpl>(); - - for (HostGroupEntity hostGroup : blueprint.getHostGroups()) { - mapHostGroups.put(hostGroup.getName(), new HostGroupImpl(hostGroup, stack, this)); - } - return mapHostGroups; - } - - /** - * Parse stack information. - * - * @param blueprint associated blueprint - * - * @return stack instance - * - * @throws SystemException an unexpected exception occurred - */ - protected Stack parseStack(BlueprintEntity blueprint) throws SystemException { - Stack stack; - try { - stack = new Stack(blueprint.getStack(), getManagementController()); - } catch (StackAccessException e) { - throw new IllegalArgumentException( - "Invalid stack information provided for cluster. " - + blueprint.getStack()); - } catch (AmbariException e) { - throw new SystemException("Unable to obtain stack information.", e); - } - return stack; - } - - /** - * Validate blueprint topology. - * An exception is thrown in the case of validation failure. - * For missing components which are auto-deploy enabled, these are added to the topology which is reflected - * in the blueprint entity that is returned. - * - * @param blueprint blueprint to validate - * - * @return blueprint entity which may have been updated as a result of auto-deployment of components. - * - * @throws AmbariException an unexpected error occurred - * @throws IllegalArgumentException when validation fails - */ - protected BlueprintEntity validateTopology(BlueprintEntity blueprint) throws AmbariException { - Stack stack = new Stack(blueprint.getStack(), getManagementController()); - Map<String, HostGroupImpl> hostGroupMap = parseBlueprintHostGroups(blueprint, stack); - Collection<HostGroupImpl> hostGroups = hostGroupMap.values(); - Map<String, Map<String, String>> clusterConfig = processBlueprintConfigurations(blueprint, null); - Map<String, Map<String, Collection<DependencyInfo>>> missingDependencies = - new HashMap<String, Map<String, Collection<DependencyInfo>>>(); - - Collection<String> services = getTopologyServices(hostGroups); - for (HostGroupImpl group : hostGroups) { - Map<String, Collection<DependencyInfo>> missingGroupDependencies = - group.validateTopology(hostGroups, services, clusterConfig); - if (! missingGroupDependencies.isEmpty()) { - missingDependencies.put(group.getEntity().getName(), missingGroupDependencies); - } - } - - Collection<String> cardinalityFailures = new HashSet<String>(); - for (String service : services) { - for (String component : stack.getComponents(service)) { - Cardinality cardinality = stack.getCardinality(component); - AutoDeployInfo autoDeploy = stack.getAutoDeployInfo(component); - if (cardinality.isAll()) { - cardinalityFailures.addAll(verifyComponentInAllHostGroups( - blueprint, hostGroups, component, autoDeploy)); - } else { - cardinalityFailures.addAll(verifyComponentCardinalityCount( - blueprint, hostGroups, component, cardinality, autoDeploy, stack, clusterConfig)); - } - } - } - - if (! missingDependencies.isEmpty() || ! cardinalityFailures.isEmpty()) { - generateInvalidTopologyException(missingDependencies, cardinalityFailures); - } - - return blueprint; - } - - /** - * Process cluster scoped configurations contained in blueprint. - * - * @param blueprint blueprint entity - * - * @return cluster scoped properties contained within in blueprint - */ - protected Map<String, Map<String, String>> processBlueprintConfigurations( - BlueprintEntity blueprint, Collection<Map<String, String>> configOverrides) { - - Map<String, Map<String, String>> mapConfigurations = new HashMap<String, Map<String, String>>(); - Collection<BlueprintConfigEntity> configs = blueprint.getConfigurations(); - Gson jsonSerializer = new Gson(); - - for (BlueprintConfigEntity config : configs) { - mapConfigurations.put(config.getType(), jsonSerializer.<Map<String, String>> fromJson( - config.getConfigData(), Map.class)); - } - overrideExistingProperties(mapConfigurations, configOverrides); - - return mapConfigurations; - } - - /** - * Process cluster scoped configuration attributes contained in blueprint. - * - * @param blueprint blueprint entity - * - * @return cluster scoped property attributes contained within in blueprint - */ - protected Map<String, Map<String, Map<String, String>>> processBlueprintAttributes(BlueprintEntity blueprint) { - - Map<String, Map<String, Map<String, String>>> mapAttributes = - new HashMap<String, Map<String, Map<String, String>>>(); - Collection<BlueprintConfigEntity> configs = blueprint.getConfigurations(); - - if (configs != null) { - Gson gson = new Gson(); - for (BlueprintConfigEntity config : configs) { - Map<String, Map<String, String>> typeAttrs = - gson.<Map<String, Map<String, String>>>fromJson(config.getConfigAttributes(), Map.class); - if (typeAttrs != null && !typeAttrs.isEmpty()) { - mapAttributes.put(config.getType(), typeAttrs); - } - } - } - - return mapAttributes; - } - - /** - * Override existing properties or add new. - * - * @param existingProperties current property values - * @param configOverrides override properties - */ - protected void overrideExistingProperties(Map<String, Map<String, String>> existingProperties, - Collection<Map<String, String>> configOverrides) { - if (configOverrides != null) { - for (Map<String, String> properties : configOverrides) { - String category = null; - int propertyOffset = -1; - for (Map.Entry<String, String> entry : properties.entrySet()) { - String absolutePropName = entry.getKey(); - if (category == null) { - propertyOffset = absolutePropName.indexOf('/'); - category = absolutePropName.substring(0, propertyOffset); - } - Map<String, String> existingCategoryProperties = existingProperties.get(category); - if (existingCategoryProperties == null) { - existingCategoryProperties = new HashMap<String, String>(); - existingProperties.put(category, existingCategoryProperties); - } - //override existing property or add new - existingCategoryProperties.put(absolutePropName.substring(propertyOffset + 1), entry.getValue()); - } - } - } - } - - /** - * Add a new component entity to a host group entity. - * - * @param blueprint blueprint entity - * @param hostGroup host group name - * @param component name of component which is being added - */ - protected void addComponentToBlueprint(BlueprintEntity blueprint, String hostGroup, String component) { - HostGroupComponentEntity componentEntity = new HostGroupComponentEntity(); - componentEntity.setBlueprintName(blueprint.getBlueprintName()); - componentEntity.setName(component); - - for (HostGroupEntity hostGroupEntity : blueprint.getHostGroups()) { - if (hostGroupEntity.getName().equals(hostGroup)) { - componentEntity.setHostGroupEntity(hostGroupEntity); - componentEntity.setHostGroupName(hostGroupEntity.getName()); - hostGroupEntity.addComponent(componentEntity); - break; - } - } - } - - /** - * Obtain a blueprint entity based on name. - * - * @param blueprintName name of blueprint to obtain - * - * @return blueprint entity for the given name - * @throws IllegalArgumentException no blueprint with the given name found - */ - protected BlueprintEntity getExistingBlueprint(String blueprintName) { - BlueprintEntity blueprint = blueprintDAO.findByName(blueprintName); - if (blueprint == null) { - throw new IllegalArgumentException("Specified blueprint doesn't exist: " + blueprintName); - } - return blueprint; - } - - /** - * Get all services provided in topology. - * - * @param hostGroups all host groups in topology - * - * @return collections of all services provided by topology - */ - protected Collection<String> getTopologyServices(Collection<HostGroupImpl> hostGroups) { - Collection<String> services = new HashSet<String>(); - for (HostGroupImpl group : hostGroups) { - services.addAll(group.getServices()); - } - return services; - } - - /** - * Determine if a component is managed, meaning that it is running inside of the cluster - * topology. Generally, non-managed dependencies will be database components. - * - * @param stack stack instance - * @param component component to determine if it is managed - * @param clusterConfig cluster configuration - * - * @return true if the specified component managed by the cluster; false otherwise - */ - protected boolean isDependencyManaged(Stack stack, String component, Map<String, Map<String, String>> clusterConfig) { - boolean isManaged = true; - String externalComponentConfig = stack.getExternalComponentConfig(component); - if (externalComponentConfig != null) { - String[] toks = externalComponentConfig.split("/"); - String externalComponentConfigType = toks[0]; - String externalComponentConfigProp = toks[1]; - Map<String, String> properties = clusterConfig.get(externalComponentConfigType); - if (properties != null && properties.containsKey(externalComponentConfigProp)) { - if (properties.get(externalComponentConfigProp).startsWith("Existing")) { - isManaged = false; - } - } - } - return isManaged; - } - - /** - * Verify that a component meets cardinality requirements. For components that are - * auto-install enabled, will add component to topology if needed. - * - * @param blueprint blueprint instance - * @param hostGroups collection of host groups - * @param component component to validate - * @param cardinality required cardinality - * @param autoDeploy auto-deploy information for component - * - * @return collection of missing component information - */ - private Collection<String> verifyComponentCardinalityCount(BlueprintEntity blueprint, - Collection<HostGroupImpl> hostGroups, - String component, - Cardinality cardinality, - AutoDeployInfo autoDeploy, - Stack stack, - Map<String, Map<String, String>> clusterConfig) { - - Collection<String> cardinalityFailures = new HashSet<String>(); - - if (BlueprintConfigurationProcessor.isNameNodeHAEnabled(clusterConfig) && - (component.equals("SECONDARY_NAMENODE"))) { - // override the cardinality for this component in an HA deployment, - // since the SECONDARY_NAMENODE should not be started in this scenario - cardinality = new Cardinality("0"); - } - - int actualCount = getHostGroupsForComponent(component, hostGroups).size(); - if (! cardinality.isValidCount(actualCount)) { - boolean validated = ! isDependencyManaged(stack, component, clusterConfig); - if (! validated && autoDeploy != null && autoDeploy.isEnabled() && cardinality.supportsAutoDeploy()) { - String coLocateName = autoDeploy.getCoLocate(); - if (coLocateName != null && ! coLocateName.isEmpty()) { - Collection<HostGroupImpl> coLocateHostGroups = getHostGroupsForComponent( - coLocateName.split("/")[1], hostGroups); - if (! coLocateHostGroups.isEmpty()) { - validated = true; - HostGroupImpl group = coLocateHostGroups.iterator().next(); - if (group.addComponent(component)) { - addComponentToBlueprint(blueprint, group.getEntity().getName(), component); - } - } - } - } - if (! validated) { - cardinalityFailures.add(component + "(actual=" + actualCount + ", required=" + - cardinality.cardinality + ")"); - } - } - return cardinalityFailures; - } - - /** - * Verify that a component is included in all host groups. - * For components that are auto-install enabled, will add component to topology if needed. - * - * @param blueprint blueprint instance - * @param hostGroups collection of host groups - * @param component component to validate - * @param autoDeploy auto-deploy information for component - * - * @return collection of missing component information - */ - private Collection<String> verifyComponentInAllHostGroups(BlueprintEntity blueprint, - Collection<HostGroupImpl> hostGroups, - String component, - AutoDeployInfo autoDeploy) { - - Collection<String> cardinalityFailures = new HashSet<String>(); - int actualCount = getHostGroupsForComponent(component, hostGroups).size(); - if (actualCount != hostGroups.size()) { - if (autoDeploy != null && autoDeploy.isEnabled()) { - for (HostGroupImpl group : hostGroups) { - if (group.addComponent(component)) { - addComponentToBlueprint(blueprint, group.getEntity().getName(), component); - } - } - } else { - cardinalityFailures.add(component + "(actual=" + actualCount + ", required=ALL)"); - } - } - return cardinalityFailures; - } - - /** - * Generate an exception for topology validation failure. - * - * @param missingDependencies missing dependency information - * @param cardinalityFailures missing service component information - * - * @throws IllegalArgumentException Always thrown and contains information regarding the topology validation failure - * in the msg - */ - private void generateInvalidTopologyException(Map<String, Map<String, Collection<DependencyInfo>>> missingDependencies, - Collection<String> cardinalityFailures) { - - String msg = "Cluster Topology validation failed."; - if (! cardinalityFailures.isEmpty()) { - msg += " Invalid service component count: " + cardinalityFailures; - } - if (! missingDependencies.isEmpty()) { - msg += " Unresolved component dependencies: " + missingDependencies; - } - msg += ". To disable topology validation and create the blueprint, " + - "add the following to the end of the url: '?validate_topology=false'"; - throw new IllegalArgumentException(msg); - } - - /** - * Create host and host_component resources. - * - * @param blueprintHostGroups host groups specified in blueprint - * @param clusterName cluster name - * - * @throws SystemException an unexpected exception occurred - * @throws UnsupportedPropertyException an invalid property was specified - * @throws ResourceAlreadyExistsException attempt to create a host or host_component which already exists - * @throws NoSuchParentResourceException a required parent resource is missing - */ - protected void createHostAndComponentResources(Map<String, HostGroupImpl> blueprintHostGroups, String clusterName) - throws SystemException, UnsupportedPropertyException, ResourceAlreadyExistsException, NoSuchParentResourceException { - - createHostAndComponentResources(blueprintHostGroups, clusterName, getResourceProvider(Resource.Type.Host)); - } - - /** - * Create host and host_component resources via the specified host resource provider. - * - * @param blueprintHostGroups host groups specified in blueprint - * @param clusterName cluster name - * @param hostProvider host resource provider - * - * @throws SystemException an unexpected exception occurred - * @throws UnsupportedPropertyException an invalid property was specified - * @throws ResourceAlreadyExistsException attempt to create a host or host_component which already exists - * @throws NoSuchParentResourceException a required parent resource is missing - */ - protected void createHostAndComponentResources(Map<String, HostGroupImpl> blueprintHostGroups, - String clusterName, - ResourceProvider hostProvider) - throws SystemException, - UnsupportedPropertyException, - ResourceAlreadyExistsException, - NoSuchParentResourceException { - - ResourceProvider hostComponentProvider = getResourceProvider(Resource.Type.HostComponent); - for (HostGroupImpl group : blueprintHostGroups.values()) { - for (String host : group.getHostInfo()) { - Map<String, Object> hostProperties = new HashMap<String, Object>(); - hostProperties.put("Hosts/cluster_name", clusterName); - hostProperties.put("Hosts/host_name", host); - - hostProvider.createResources(new RequestImpl( - null, Collections.singleton(hostProperties), null, null)); - - // create clusters/hosts/host_components - Set<Map<String, Object>> setHostComponentRequestProps = new HashSet<Map<String, Object>>(); - for (String hostComponent : group.getComponents()) { - // AMBARI_SERVER is not recognized by Ambari as a component - if (! hostComponent.equals("AMBARI_SERVER")) { - Map<String, Object> hostComponentProperties = new HashMap<String, Object>(); - hostComponentProperties.put("HostRoles/cluster_name", clusterName); - hostComponentProperties.put("HostRoles/host_name", host); - hostComponentProperties.put("HostRoles/component_name", hostComponent); - setHostComponentRequestProps.add(hostComponentProperties); - } - } - hostComponentProvider.createResources(new RequestImpl( - null, setHostComponentRequestProps, null, null)); - } - } - } - - /** - * Get a config group name based on a bp and host group. - * - * @param bpName blueprint name - * @param hostGroupName host group name - * @return config group name - */ - protected String getConfigurationGroupName(String bpName, String hostGroupName) { - return String.format("%s:%s", bpName, hostGroupName); - } - - - // ----- Inner Classes ----------------------------------------------------- - - /** - * Host group representation. - */ - protected static class HostGroupImpl implements HostGroup { - /** - * Host group entity - */ - private HostGroupEntity hostGroup; - - /** - * Components contained in the host group - */ - private Collection<String> components = new HashSet<String>(); - - /** - * Hosts contained associated with the host group - */ - private Collection<String> hosts = new HashSet<String>(); - - /** - * Map of service to components for the host group - */ - private Map<String, Set<String>> componentsForService = new HashMap<String, Set<String>>(); - - /** - * Map of host group configurations. - * Type -> Map<Key, Val> - */ - private Map<String, Map<String, String>> configurations = - new HashMap<String, Map<String, String>>(); - - /** - * Associated stack - */ - private Stack stack; - - /** - * The Blueprint processor associated with this HostGroupImpl instance - */ - private final BaseBlueprintProcessor blueprintProcessor; - - /** - * Constructor. - * - * @param hostGroup host group - * @param stack stack - */ - public HostGroupImpl(HostGroupEntity hostGroup, Stack stack, BaseBlueprintProcessor blueprintProcessor) { - this.hostGroup = hostGroup; - this.stack = stack; - this.blueprintProcessor = blueprintProcessor; - parseComponents(); - parseConfigurations(); - } - - @Override - public String getName() { - return hostGroup.getName(); - } - - @Override - public Collection<String> getComponents() { - return components; - } - - @Override - public Collection<String> getHostInfo() { - return hosts; - } - - /** - * Associate a host with the host group. - * - * @param fqdn fully qualified domain name of the host being added - */ - public void addHostInfo(String fqdn) { - hosts.add(fqdn); - } - - /** - * Get the services which are deployed to this host group. - * - * @return collection of services which have components in this host group - */ - public Collection<String> getServices() { - return componentsForService.keySet(); - } - - /** - * Add a component to the host group. - * - * @param component component to add - * - * @return true if component was added; false if component already existed - */ - public boolean addComponent(String component) { - boolean added = components.add(component); - if (added) { - String service = stack.getServiceForComponent(component); - if (service != null) { - // an example of a component without a service in the stack is AMBARI_SERVER - Set<String> serviceComponents = componentsForService.get(service); - if (serviceComponents == null) { - serviceComponents = new HashSet<String>(); - componentsForService.put(service, serviceComponents); - } - serviceComponents.add(component); - } - } - return added; - } - - /** - * Get the components for the specified service which are associated with the host group. - * - * @param service service name - * - * @return set of component names - */ - public Collection<String> getComponents(String service) { - return componentsForService.get(service); - } - - /** - * Get the configurations associated with the host group. - * - * @return map of configuration type to a map of properties - */ - @Override - public Map<String, Map<String, String>> getConfigurationProperties() { - return configurations; - } - - /** - * Get the associated entity. - * - * @return associated host group entity - */ - public HostGroupEntity getEntity() { - return hostGroup; - } - - /** - * Validate host group topology. This includes ensuring that all component dependencies are satisfied. - * - * @param hostGroups collection of all host groups - * @param services set of services in cluster topology - * @param clusterConfig cluster configuration - * - * @return map of component to missing dependencies - */ - public Map<String, Collection<DependencyInfo>> validateTopology(Collection<HostGroupImpl> hostGroups, - Collection<String> services, - Map<String, Map<String, String>> clusterConfig) { - - Map<String, Collection<DependencyInfo>> missingDependencies = - new HashMap<String, Collection<DependencyInfo>>(); - - for (String component : new HashSet<String>(components)) { - Collection<DependencyInfo> dependenciesForComponent = stack.getDependenciesForComponent(component); - for (DependencyInfo dependency : dependenciesForComponent) { - String conditionalService = stack.getConditionalServiceForDependency(dependency); - if (conditionalService != null && ! services.contains(conditionalService)) { - continue; - } - - BlueprintEntity entity = hostGroup.getBlueprintEntity(); - String dependencyScope = dependency.getScope(); - String componentName = dependency.getComponentName(); - AutoDeployInfo autoDeployInfo = dependency.getAutoDeploy(); - boolean resolved = false; - - if (dependencyScope.equals("cluster")) { - Collection<String> missingDependencyInfo = blueprintProcessor.verifyComponentCardinalityCount(entity, hostGroups, - componentName, new Cardinality("1+"), autoDeployInfo, stack, clusterConfig); - resolved = missingDependencyInfo.isEmpty(); - } else if (dependencyScope.equals("host")) { - if (components.contains(component) || (autoDeployInfo != null && autoDeployInfo.isEnabled())) { - resolved = true; - if (addComponent(componentName)) { - blueprintProcessor.addComponentToBlueprint(hostGroup.getBlueprintEntity(), getEntity().getName(), componentName); - } - } - } - - if (! resolved) { - Collection<DependencyInfo> missingCompDependencies = missingDependencies.get(component); - if (missingCompDependencies == null) { - missingCompDependencies = new HashSet<DependencyInfo>(); - missingDependencies.put(component, missingCompDependencies); - } - missingCompDependencies.add(dependency); - } - } - } - return missingDependencies; - } - - /** - * Parse component information. - */ - private void parseComponents() { - for (HostGroupComponentEntity componentEntity : hostGroup.getComponents() ) { - addComponent(componentEntity.getName()); - } - } - - /** - * Parse host group configurations. - */ - private void parseConfigurations() { - Gson jsonSerializer = new Gson(); - for (HostGroupConfigEntity configEntity : hostGroup.getConfigurations()) { - String type = configEntity.getType(); - Map<String, String> typeProperties = configurations.get(type); - if ( typeProperties == null) { - typeProperties = new HashMap<String, String>(); - configurations.put(type, typeProperties); - } - Map<String, String> propertyMap = jsonSerializer.<Map<String, String>>fromJson( - configEntity.getConfigData(), Map.class); - - if (propertyMap != null) { - typeProperties.putAll(propertyMap); - } - } - } - } - -}
http://git-wip-us.apache.org/repos/asf/ambari/blob/c9f0dd0b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java index 9c3266a..95e9807 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java @@ -19,12 +19,20 @@ package org.apache.ambari.server.controller.internal; +import org.apache.ambari.server.topology.Cardinality; +import org.apache.ambari.server.topology.ClusterTopology; +import org.apache.ambari.server.topology.Configuration; +import org.apache.ambari.server.topology.HostGroupInfo; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; +import java.util.Iterator; import java.util.LinkedHashSet; import java.util.LinkedList; import java.util.List; @@ -39,6 +47,8 @@ import java.util.regex.Pattern; */ public class BlueprintConfigurationProcessor { + protected final static Logger LOG = LoggerFactory.getLogger(BlueprintConfigurationProcessor.class); + /** * Single host topology updaters */ @@ -95,33 +105,61 @@ public class BlueprintConfigurationProcessor { private static Set<String> configPropertiesWithHASupport = new HashSet<String>(Arrays.asList("fs.defaultFS", "hbase.rootdir", "instance.volumes")); - - /** * Configuration properties to be updated */ - private Map<String, Map<String, String>> properties; + //private Map<String, Map<String, String>> properties; + private ClusterTopology clusterTopology; - /** - * Constructor. - * - * @param properties properties to update - */ - public BlueprintConfigurationProcessor(Map<String, Map<String, String>> properties) { - this.properties = properties; + + public BlueprintConfigurationProcessor(ClusterTopology clusterTopology) { + this.clusterTopology = clusterTopology; + } + + public Collection<String> getRequiredHostGroups() { + Collection<String> requiredHostGroups = new HashSet<String>(); + + for (Map<String, Map<String, PropertyUpdater>> updaterMap : createCollectionOfUpdaters()) { + for (Map.Entry<String, Map<String, PropertyUpdater>> entry : updaterMap.entrySet()) { + String type = entry.getKey(); + for (Map.Entry<String, PropertyUpdater> updaterEntry : entry.getValue().entrySet()) { + String propertyName = updaterEntry.getKey(); + PropertyUpdater updater = updaterEntry.getValue(); + + // topo cluster scoped configuration which also includes all default and BP properties + Map<String, Map<String, String>> clusterProps = clusterTopology.getConfiguration().getFullProperties(); + Map<String, String> typeMap = clusterProps.get(type); + if (typeMap != null && typeMap.containsKey(propertyName)) { + requiredHostGroups.addAll(updater.getRequiredHostGroups( + typeMap.get(propertyName), clusterProps, clusterTopology)); + } + + // host group configs + for (HostGroupInfo groupInfo : clusterTopology.getHostGroupInfo().values()) { + Map<String, Map<String, String>> hgConfigProps = groupInfo.getConfiguration().getProperties(); + Map<String, String> hgTypeMap = hgConfigProps.get(type); + if (hgTypeMap != null && hgTypeMap.containsKey(propertyName)) { + requiredHostGroups.addAll(updater.getRequiredHostGroups( + hgTypeMap.get(propertyName), hgConfigProps, clusterTopology)); + } + } + } + } + } + return requiredHostGroups; } + /** * Update properties for cluster creation. This involves updating topology related properties with * concrete topology information. - * - * @param hostGroups host groups of cluster to be deployed - * @param stackDefinition stack used for cluster creation - * - * @return updated properties */ - public Map<String, Map<String, String>> doUpdateForClusterCreate(Map<String, ? extends HostGroup> hostGroups, Stack stackDefinition) { + public void doUpdateForClusterCreate() throws ConfigurationTopologyException { + Configuration clusterConfig = clusterTopology.getConfiguration(); + Map<String, Map<String, String>> clusterProps = clusterConfig.getFullProperties(); + Map<String, HostGroupInfo> groupInfoMap = clusterTopology.getHostGroupInfo(); + for (Map<String, Map<String, PropertyUpdater>> updaterMap : createCollectionOfUpdaters()) { for (Map.Entry<String, Map<String, PropertyUpdater>> entry : updaterMap.entrySet()) { String type = entry.getKey(); @@ -129,45 +167,73 @@ public class BlueprintConfigurationProcessor { String propertyName = updaterEntry.getKey(); PropertyUpdater updater = updaterEntry.getValue(); - Map<String, String> typeMap = properties.get(type); + // topo cluster scoped configuration which also includes all default and BP properties + Map<String, String> typeMap = clusterProps.get(type); if (typeMap != null && typeMap.containsKey(propertyName)) { - typeMap.put(propertyName, updater.updateForClusterCreate( - hostGroups, propertyName, typeMap.get(propertyName), properties, stackDefinition)); + clusterConfig.setProperty(type, propertyName, updater.updateForClusterCreate( + propertyName, typeMap.get(propertyName), clusterProps, clusterTopology)); + } + + // host group configs + for (HostGroupInfo groupInfo : groupInfoMap.values()) { + Configuration hgConfig = groupInfo.getConfiguration(); + Map<String, Map<String, String>> hgConfigProps = hgConfig.getProperties(); + Map<String, String> hgTypeMap = hgConfigProps.get(type); + if (hgTypeMap != null && hgTypeMap.containsKey(propertyName)) { + hgConfig.setProperty(type, propertyName, updater.updateForClusterCreate( + propertyName, hgTypeMap.get(propertyName), hgConfigProps, clusterTopology)); + } } } } } - if (isNameNodeHAEnabled()) { + //todo: lots of hard coded HA rules included here + if (clusterTopology.isNameNodeHAEnabled()) { // if the active/stanbdy namenodes are not specified, assign them automatically - if (! isNameNodeHAInitialActiveNodeSet(properties) && ! isNameNodeHAInitialStandbyNodeSet(properties)) { - Collection<HostGroup> listOfHostGroups = new LinkedList<HostGroup>(); - for (String key : hostGroups.keySet()) { - listOfHostGroups.add(hostGroups.get(key)); + if (! isNameNodeHAInitialActiveNodeSet(clusterProps) && ! isNameNodeHAInitialStandbyNodeSet(clusterProps)) { + Collection<String> nnHosts = clusterTopology.getHostAssignmentsForComponent("NAMENODE"); + if (nnHosts.size() != 2) { + throw new ConfigurationTopologyException("NAMENODE HA requires exactly 2 hosts running NAMENODE but there are: " + + nnHosts.size() + " Hosts: " + nnHosts); } - Collection<HostGroup> hostGroupsContainingNameNode = - getHostGroupsForComponent("NAMENODE", listOfHostGroups); // set the properties that configure which namenode is active, // and which is a standby node in this HA deployment - Map<String, String> hadoopEnv = properties.get("hadoop-env"); - if (hostGroupsContainingNameNode.size() == 2) { - List<HostGroup> listOfGroups = new LinkedList<HostGroup>(hostGroupsContainingNameNode); - hadoopEnv.put("dfs_ha_initial_namenode_active", listOfGroups.get(0).getHostInfo().iterator().next()); - hadoopEnv.put("dfs_ha_initial_namenode_standby", listOfGroups.get(1).getHostInfo().iterator().next()); - } else { - // handle the case where multiple hosts are mapped to an HA host group - if (hostGroupsContainingNameNode.size() == 1) { - List<String> listOfInfo = new LinkedList<String>(hostGroupsContainingNameNode.iterator().next().getHostInfo()); - // there should only be two host names that can include a NameNode install/deployment - hadoopEnv.put("dfs_ha_initial_namenode_active", listOfInfo.get(0)); - hadoopEnv.put("dfs_ha_initial_namenode_standby", listOfInfo.get(1)); - } - } + Iterator<String> nnHostIterator = nnHosts.iterator(); + clusterConfig.setProperty("hadoop-env", "dfs_ha_initial_namenode_active", nnHostIterator.next()); + clusterConfig.setProperty("hadoop-env", "dfs_ha_initial_namenode_standby", nnHostIterator.next()); } } + setMissingConfigurations(clusterProps); + } + + /** + * Update properties for blueprint export. + * This involves converting concrete topology information to host groups. + */ + //todo: use cluster topology + public void doUpdateForBlueprintExport() { + + // HA configs are only processed in cluster configuration, not HG configurations + if (clusterTopology.isNameNodeHAEnabled()) { + doNameNodeHAUpdate(); + } + Collection<Map<String, Map<String, String>>> allConfigs = new ArrayList<Map<String, Map<String, String>>>(); + allConfigs.add(clusterTopology.getConfiguration().getFullProperties()); + for (HostGroupInfo groupInfo : clusterTopology.getHostGroupInfo().values()) { + // don't use full properties, only the properties specified in the host group config + allConfigs.add(groupInfo.getConfiguration().getProperties()); + } + + for (Map<String, Map<String, String>> properties : allConfigs) { + doSingleHostExportUpdate(singleHostTopologyUpdaters, properties); + doSingleHostExportUpdate(dbHostTopologyUpdaters, properties); - return properties; + doMultiHostExportUpdate(multiHostTopologyUpdaters, properties); + + doRemovePropertyExport(removePropertyUpdaters, properties); + } } /** @@ -179,7 +245,7 @@ public class BlueprintConfigurationProcessor { * @return Collection of PropertyUpdater maps used to handle cluster config update */ private Collection<Map<String, Map<String, PropertyUpdater>>> createCollectionOfUpdaters() { - return (isNameNodeHAEnabled()) ? addHAUpdaters(allUpdaters) : allUpdaters; + return (clusterTopology.isNameNodeHAEnabled()) ? addHAUpdaters(allUpdaters) : allUpdaters; } /** @@ -208,29 +274,6 @@ public class BlueprintConfigurationProcessor { } /** - * Update properties for blueprint export. - * This involves converting concrete topology information to host groups. - * - * @param hostGroups cluster host groups - * - * @return updated properties - */ - public Map<String, Map<String, String>> doUpdateForBlueprintExport(Collection<? extends HostGroup> hostGroups) { - doSingleHostExportUpdate(hostGroups, singleHostTopologyUpdaters); - doSingleHostExportUpdate(hostGroups, dbHostTopologyUpdaters); - - if (isNameNodeHAEnabled()) { - doNameNodeHAUpdate(hostGroups); - } - - doMultiHostExportUpdate(hostGroups, multiHostTopologyUpdaters); - - doRemovePropertyExport(removePropertyUpdaters); - - return properties; - } - - /** * Performs export update for the set of properties that do not * require update during cluster setup, but should be removed * during a Blueprint export. @@ -243,7 +286,9 @@ public class BlueprintConfigurationProcessor { * @param updaters set of updaters for properties that should * always be removed during a Blueprint export */ - private void doRemovePropertyExport(Map<String, Map<String, PropertyUpdater>> updaters) { + private void doRemovePropertyExport(Map<String, Map<String, PropertyUpdater>> updaters, + Map<String, Map<String, String>> properties) { + for (Map.Entry<String, Map<String, PropertyUpdater>> entry : updaters.entrySet()) { String type = entry.getKey(); for (String propertyName : entry.getValue().keySet()) { @@ -261,14 +306,13 @@ public class BlueprintConfigurationProcessor { * dynamically determines the property names, and registers PropertyUpdaters to handle the masking of * host names in these configuration items. * - * @param hostGroups cluster host groups */ - public void doNameNodeHAUpdate(Collection<? extends HostGroup> hostGroups) { + public void doNameNodeHAUpdate() { Map<String, Map<String, PropertyUpdater>> highAvailabilityUpdaters = createMapOfHAUpdaters(); // perform a single host update on these dynamically generated property names if (highAvailabilityUpdaters.get("hdfs-site").size() > 0) { - doSingleHostExportUpdate(hostGroups, highAvailabilityUpdaters); + doSingleHostExportUpdate(highAvailabilityUpdaters, clusterTopology.getConfiguration().getFullProperties()); } } @@ -286,7 +330,8 @@ public class BlueprintConfigurationProcessor { Map<String, PropertyUpdater> hdfsSiteUpdatersForAvailability = new HashMap<String, PropertyUpdater>(); highAvailabilityUpdaters.put("hdfs-site", hdfsSiteUpdatersForAvailability); - Map<String, String> hdfsSiteConfig = properties.get("hdfs-site"); + //todo: Do we need to call this for HG configurations? + Map<String, String> hdfsSiteConfig = clusterTopology.getConfiguration().getFullProperties().get("hdfs-site"); // generate the property names based on the current HA config for the NameNode deployments for (String nameService : parseNameServices(hdfsSiteConfig)) { for (String nameNode : parseNameNodes(nameService, hdfsSiteConfig)) { @@ -302,27 +347,6 @@ public class BlueprintConfigurationProcessor { } /** - * Convenience function to determine if NameNode HA is enabled. - * - * @return true if NameNode HA is enabled - * false if NameNode HA is not enabled - */ - boolean isNameNodeHAEnabled() { - return isNameNodeHAEnabled(properties); - } - - /** - * Static convenience function to determine if NameNode HA is enabled - * @param configProperties configuration properties for this cluster - * @return true if NameNode HA is enabled - * false if NameNode HA is not enabled - */ - static boolean isNameNodeHAEnabled(Map<String, Map<String, String>> configProperties) { - return configProperties.containsKey("hdfs-site") && configProperties.get("hdfs-site").containsKey("dfs.nameservices"); - } - - - /** * Static convenience function to determine if Yarn ResourceManager HA is enabled * @param configProperties configuration properties for this cluster * @return true if Yarn ResourceManager HA is enabled @@ -413,11 +437,9 @@ public class BlueprintConfigurationProcessor { /** * Update single host topology configuration properties for blueprint export. * - * @param hostGroups cluster export * @param updaters registered updaters */ - private void doSingleHostExportUpdate(Collection<? extends HostGroup> hostGroups, - Map<String, Map<String, PropertyUpdater>> updaters) { + private void doSingleHostExportUpdate(Map<String, Map<String, PropertyUpdater>> updaters, Map<String, Map<String, String>> properties) { for (Map.Entry<String, Map<String, PropertyUpdater>> entry : updaters.entrySet()) { String type = entry.getKey(); @@ -427,14 +449,15 @@ public class BlueprintConfigurationProcessor { Map<String, String> typeProperties = properties.get(type); if (typeProperties != null && typeProperties.containsKey(propertyName)) { String propValue = typeProperties.get(propertyName); - for (HostGroup group : hostGroups) { - Collection<String> hosts = group.getHostInfo(); + + for (HostGroupInfo groupInfo : clusterTopology.getHostGroupInfo().values()) { + Collection<String> hosts = groupInfo.getHostNames(); for (String host : hosts) { //todo: need to use regular expression to avoid matching a host which is a superset. if (propValue.contains(host)) { matchedHost = true; typeProperties.put(propertyName, propValue.replace( - host, "%HOSTGROUP::" + group.getName() + "%")); + host, "%HOSTGROUP::" + groupInfo.getHostGroupName() + "%")); break; } } @@ -501,22 +524,20 @@ public class BlueprintConfigurationProcessor { /** * Update multi host topology configuration properties for blueprint export. * - * @param hostGroups cluster host groups * @param updaters registered updaters */ - private void doMultiHostExportUpdate(Collection<? extends HostGroup> hostGroups, - Map<String, Map<String, PropertyUpdater>> updaters) { - + private void doMultiHostExportUpdate(Map<String, Map<String, PropertyUpdater>> updaters, Map<String, Map<String, String>> properties) { for (Map.Entry<String, Map<String, PropertyUpdater>> entry : updaters.entrySet()) { String type = entry.getKey(); for (String propertyName : entry.getValue().keySet()) { Map<String, String> typeProperties = properties.get(type); if (typeProperties != null && typeProperties.containsKey(propertyName)) { String propValue = typeProperties.get(propertyName); - for (HostGroup group : hostGroups) { - Collection<String> hosts = group.getHostInfo(); + for (HostGroupInfo groupInfo : clusterTopology.getHostGroupInfo().values()) { + Collection<String> hosts = groupInfo.getHostNames(); for (String host : hosts) { - propValue = propValue.replaceAll(host + "\\b", "%HOSTGROUP::" + group.getName() + "%"); + propValue = propValue.replaceAll(host + "\\b", "%HOSTGROUP::" + + groupInfo.getHostGroupName() + "%"); } } Collection<String> addedGroups = new HashSet<String>(); @@ -550,35 +571,16 @@ public class BlueprintConfigurationProcessor { } /** - * Get host groups which contain a component. - * - * @param component component name - * @param hostGroups collection of host groups to check - * - * @return collection of host groups which contain the specified component - */ - private static Collection<HostGroup> getHostGroupsForComponent(String component, - Collection<? extends HostGroup> hostGroups) { - - Collection<HostGroup> resultGroups = new LinkedHashSet<HostGroup>(); - for (HostGroup group : hostGroups ) { - if (group.getComponents().contains(component)) { - resultGroups.add(group); - } - } - return resultGroups; - } - - /** * Convert a property value which includes a host group topology token to a physical host. * - * @param hostGroups cluster host groups - * @param val value to be converted + * + * @param val value to be converted + * @param topology cluster topology * * @return updated value with physical host name */ - private static Collection<String> getHostStrings(Map<String, ? extends HostGroup> hostGroups, - String val) { + //todo: replace this with parseHostGroupToken which would return a hostgroup or null + private static Collection<String> getHostStrings(String val, ClusterTopology topology) { Collection<String> hosts = new LinkedHashSet<String>(); Matcher m = HOSTGROUP_PORT_REGEX.matcher(val); @@ -586,13 +588,13 @@ public class BlueprintConfigurationProcessor { String groupName = m.group(1); String port = m.group(2); - - HostGroup hostGroup = hostGroups.get(groupName); - if (hostGroup == null) { + HostGroupInfo hostGroupInfo = topology.getHostGroupInfo().get(groupName); + if (hostGroupInfo == null) { throw new IllegalArgumentException( "Unable to match blueprint host group token to a host group: " + groupName); } - for (String host : hostGroup.getHostInfo()) { + + for (String host : hostGroupInfo.getHostNames()) { if (port != null) { host += ":" + port; } @@ -627,26 +629,28 @@ public class BlueprintConfigurationProcessor { /** * Update a property value. * - * - * @param hostGroups host groups - * @param propertyName name of property + * @param propertyName name of * @param origValue original value of property * @param properties all properties - * @param stackDefinition definition of stack used for this cluster - * creation attempt + * @param topology cluster topology * * @return new property value */ - public String updateForClusterCreate(Map<String, ? extends HostGroup> hostGroups, - String propertyName, String origValue, Map<String, Map<String, String>> properties, Stack stackDefinition - ); + public String updateForClusterCreate(String propertyName, + String origValue, + Map<String, Map<String, String>> properties, + ClusterTopology topology); + + public Collection<String> getRequiredHostGroups(String origValue, + Map<String, Map<String, String>> properties, + ClusterTopology topology); } /** * Topology based updater which replaces the original host name of a property with the host name * which runs the associated (master) component in the new cluster. */ - static class SingleHostTopologyUpdater implements PropertyUpdater { + private static class SingleHostTopologyUpdater implements PropertyUpdater { /** * Component name */ @@ -664,43 +668,52 @@ public class BlueprintConfigurationProcessor { /** * Update the property with the new host name which runs the associated component. * - * - * @param hostGroups host groups - * @param propertyName name of property - * @param origValue original value of property - * @param properties all properties - * @param stackDefinition stack used for cluster creation + * @param propertyName name of property + * @param origValue original value of property + * @param properties all properties + * @param topology cluster topology * * @return updated property value with old host name replaced by new host name */ @Override - public String updateForClusterCreate(Map<String, ? extends HostGroup> hostGroups, - String propertyName, + public String updateForClusterCreate(String propertyName, String origValue, Map<String, Map<String, String>> properties, - Stack stackDefinition) { + ClusterTopology topology) { + //todo: getHostStrings Matcher m = HOSTGROUP_REGEX.matcher(origValue); if (m.find()) { String hostGroupName = m.group(1); - HostGroup hostGroup = hostGroups.get(hostGroupName); - //todo: ensure > 0 hosts (is this necessary) - return origValue.replace(m.group(0), hostGroup.getHostInfo().iterator().next()); + + HostGroupInfo groupInfo = topology.getHostGroupInfo().get(hostGroupName); + if (groupInfo == null) { + //todo: this should be validated in configuration validation + throw new RuntimeException( + "Encountered a host group token in configuration which couldn't be matched to a host group: " + + hostGroupName); + } + + //todo: warn if > hosts + return origValue.replace(m.group(0), groupInfo.getHostNames().iterator().next()); } else { - Collection<HostGroup> matchingGroups = getHostGroupsForComponent(component, hostGroups.values()); - if (matchingGroups.size() == 1) { - return origValue.replace("localhost", matchingGroups.iterator().next().getHostInfo().iterator().next()); + int matchingGroupCount = topology.getHostGroupsForComponent(component).size(); + if (matchingGroupCount == 1) { + Collection<String> componentHosts = topology.getHostAssignmentsForComponent(component); + //todo: warn if > 1 hosts + return origValue.replace("localhost", componentHosts.iterator().next()); } else { - Cardinality cardinality = stackDefinition.getCardinality(component); + //todo: extract all hard coded HA logic + Cardinality cardinality = topology.getBlueprint().getStack().getCardinality(component); // if no matching host groups are found for a component whose configuration // is handled by this updater, check the stack first to determine if // zero is a valid cardinality for this component. This is necessary // in the case of a component in "technical preview" status, since it // may be valid to have 0 or 1 instances of such a component in the cluster - if (matchingGroups.isEmpty() && cardinality.isValidCount(0)) { + if (matchingGroupCount == 0 && cardinality.isValidCount(0)) { return origValue; } else { - if (isNameNodeHAEnabled(properties) && isComponentNameNode() && (matchingGroups.size() == 2)) { + if (topology.isNameNodeHAEnabled() && isComponentNameNode() && (matchingGroupCount == 2)) { // if this is the defaultFS property, it should reflect the nameservice name, // rather than a hostname (used in non-HA scenarios) if (properties.containsKey("core-site") && properties.get("core-site").get("fs.defaultFS").equals(origValue)) { @@ -726,34 +739,34 @@ public class BlueprintConfigurationProcessor { } - if (isNameNodeHAEnabled(properties) && isComponentSecondaryNameNode() && (matchingGroups.isEmpty())) { + if (topology.isNameNodeHAEnabled() && isComponentSecondaryNameNode() && (matchingGroupCount == 0)) { // if HDFS HA is enabled, then no replacement is necessary for properties that refer to the SECONDARY_NAMENODE // eventually this type of information should be encoded in the stacks return origValue; } - if (isYarnResourceManagerHAEnabled(properties) && isComponentResourceManager() && (matchingGroups.size() == 2)) { + if (isYarnResourceManagerHAEnabled(properties) && isComponentResourceManager() && (matchingGroupCount == 2)) { if (!origValue.contains("localhost")) { // if this Yarn property is a FQDN, then simply return it return origValue; } } - if ((isOozieServerHAEnabled(properties)) && isComponentOozieServer() && (matchingGroups.size() > 1)) { + if ((isOozieServerHAEnabled(properties)) && isComponentOozieServer() && (matchingGroupCount > 1)) { if (!origValue.contains("localhost")) { // if this Oozie property is a FQDN, then simply return it return origValue; } } - if ((isHiveServerHAEnabled(properties)) && isComponentHiveServer() && (matchingGroups.size() > 1)) { + if ((isHiveServerHAEnabled(properties)) && isComponentHiveServer() && (matchingGroupCount > 1)) { if (!origValue.contains("localhost")) { // if this Hive property is a FQDN, then simply return it return origValue; } } - if ((isComponentHiveMetaStoreServer()) && matchingGroups.size() > 1) { + if ((isComponentHiveMetaStoreServer()) && matchingGroupCount > 1) { if (!origValue.contains("localhost")) { // if this Hive MetaStore property is a FQDN, then simply return it return origValue; @@ -768,6 +781,47 @@ public class BlueprintConfigurationProcessor { } } + @Override + public Collection<String> getRequiredHostGroups(String origValue, + Map<String, Map<String, String>> properties, + ClusterTopology topology) { + //todo: getHostStrings + Matcher m = HOSTGROUP_REGEX.matcher(origValue); + if (m.find()) { + String hostGroupName = m.group(1); + return Collections.singleton(hostGroupName); + } else { + Collection<String> matchingGroups = topology.getHostGroupsForComponent(component); + if (matchingGroups.size() == 1) { + return Collections.singleton(matchingGroups.iterator().next()); + } else { + if (topology.isNameNodeHAEnabled() && isComponentNameNode() && (matchingGroups.size() == 2)) { + // if this is the defaultFS property, it should reflect the nameservice name, + // rather than a hostname (used in non-HA scenarios) + if (properties.containsKey("core-site") && properties.get("core-site").get("fs.defaultFS").equals(origValue)) { + return Collections.emptySet(); + } + + if (properties.containsKey("hbase-site") && properties.get("hbase-site").get("hbase.rootdir").equals(origValue)) { + // hbase-site's reference to the namenode is handled differently in HA mode, since the + // reference must point to the logical nameservice, rather than an individual namenode + return Collections.emptySet(); + } + } + + if (topology.isNameNodeHAEnabled() && isComponentSecondaryNameNode() && (matchingGroups.isEmpty())) { + // if HDFS HA is enabled, then no replacement is necessary for properties that refer to the SECONDARY_NAMENODE + // eventually this type of information should be encoded in the stacks + return Collections.emptySet(); + } + + //todo: + throw new IllegalArgumentException("Unable to determine required host groups for component. " + + "Component '" + component + "' is not mapped to any host group or is mapped to multiple groups."); + } + } + } + /** * Utility method to determine if the component associated with this updater * instance is an HDFS NameNode @@ -863,14 +917,29 @@ public class BlueprintConfigurationProcessor { } @Override - public String updateForClusterCreate(Map<String, ? extends HostGroup> hostGroups, String propertyName, String origValue, Map<String, Map<String, String>> properties, Stack stackDefinition) { + public String updateForClusterCreate(String propertyName, + String origValue, + Map<String, Map<String, String>> properties, + ClusterTopology topology) { try { - return super.updateForClusterCreate(hostGroups, propertyName, origValue, properties, stackDefinition); + return super.updateForClusterCreate(propertyName, origValue, properties, topology); } catch (IllegalArgumentException illegalArgumentException) { // return the original value, since the optional component is not available in this cluster return origValue; } } + + @Override + public Collection<String> getRequiredHostGroups(String origValue, + Map<String, Map<String, String>> properties, + ClusterTopology topology) { + + try { + return super.getRequiredHostGroups(origValue, properties, topology); + } catch (IllegalArgumentException e) { + return Collections.emptySet(); + } + } } /** @@ -908,23 +977,23 @@ public class BlueprintConfigurationProcessor { * runs the associated component. If the database is external (non-managed), return the * original value. * - * - * @param hostGroups host groups - * @param origValue original value of property - * @param properties all properties - * @param stackDefinition stack used for cluster creation + * @param propertyName property name + * @param origValue original value of property + * @param properties all properties + * @param topology cluster topology * * @return updated property value with old host name replaced by new host name or original value * if the database is external */ @Override - public String updateForClusterCreate(Map<String, ? extends HostGroup> hostGroups, - String propertyName, - String origValue, Map<String, Map<String, String>> properties, - Stack stackDefinition) { + + public String updateForClusterCreate(String propertyName, + String origValue, + Map<String, Map<String, String>> properties, + ClusterTopology topology) { if (isDatabaseManaged(properties)) { - return super.updateForClusterCreate(hostGroups, propertyName, origValue, properties, stackDefinition); + return super.updateForClusterCreate(propertyName, origValue, properties, topology); } else { return origValue; } @@ -948,7 +1017,6 @@ public class BlueprintConfigurationProcessor { */ private static class MultipleHostTopologyUpdater implements PropertyUpdater { - private static final Character DEFAULT_SEPARATOR = ','; /** @@ -996,35 +1064,32 @@ public class BlueprintConfigurationProcessor { * Update all host names included in the original property value with new host names which run the associated * component. * - * - * @param hostGroups host groups - * - * @param origValue original value of property - * @param properties all properties - * @param stackDefinition stack used for cluster creation - * + * @param propertyName property name + * @param origValue original value of property + * @param properties all properties + * @param topology cluster topology * @return updated property value with old host names replaced by new host names */ @Override - public String updateForClusterCreate(Map<String, ? extends HostGroup> hostGroups, - String propertyName, + public String updateForClusterCreate(String propertyName, String origValue, Map<String, Map<String, String>> properties, - Stack stackDefinition) { + ClusterTopology topology) { + StringBuilder sb = new StringBuilder(); if (!origValue.contains("%HOSTGROUP") && - (!origValue.contains("localhost"))) { + (!origValue.contains("localhost"))) { // this property must contain FQDNs specified directly by the user // of the Blueprint, so the processor should not attempt to update them return origValue; } String prefix = null; - Collection<String> hostStrings = getHostStrings(hostGroups, origValue); + Collection<String> hostStrings = getHostStrings(origValue, topology); if (hostStrings.isEmpty()) { //default non-exported original value - String port = null; + String port; for (String urlScheme : setOfKnownURLSchemes) { if (origValue.startsWith(urlScheme)) { prefix = urlScheme; @@ -1039,22 +1104,15 @@ public class BlueprintConfigurationProcessor { port = calculatePort(origValue); } - - Collection<HostGroup> matchingGroups = getHostGroupsForComponent(component, hostGroups.values()); - for (HostGroup group : matchingGroups) { - for (String host : group.getHostInfo()) { - if (port != null) { - host += ":" + port; - } - hostStrings.add(host); + for (String host : topology.getHostAssignmentsForComponent(component)) { + if (port != null) { + host += ":" + port; } + hostStrings.add(host); } } - - String suffix = null; - // parse out prefix if one exists Matcher matcher = HOSTGROUP_PORT_REGEX.matcher(origValue); if (matcher.find()) { @@ -1075,7 +1133,6 @@ public class BlueprintConfigurationProcessor { if ((indexOfEnd > -1) && (indexOfEnd < (origValue.length() - 1))) { suffix = origValue.substring(indexOfEnd); } - } // add hosts to property, using the specified separator @@ -1090,17 +1147,12 @@ public class BlueprintConfigurationProcessor { } else { firstHost = false; } - - - sb.append(host); } if ((suffix != null) && (!suffix.equals("']"))) { sb.append(suffix); } - - return sb.toString(); } @@ -1112,6 +1164,34 @@ public class BlueprintConfigurationProcessor { return null; } + + @Override + public Collection<String> getRequiredHostGroups(String origValue, + Map<String, Map<String, String>> properties, + ClusterTopology topology) { + + Collection<String> requiredHostGroups = new HashSet<String>(); + + // add all host groups specified in host group tokens + Matcher m = HOSTGROUP_PORT_REGEX.matcher(origValue); + while (m.find()) { + String groupName = m.group(1); + + if (!topology.getBlueprint().getHostGroups().containsKey(groupName)) { + throw new IllegalArgumentException( + "Unable to match blueprint host group token to a host group: " + groupName); + } + requiredHostGroups.add(groupName); + } + + //todo: for now assuming that we will either have HG tokens or standard replacement but not both + //todo: as is done in updateForClusterCreate + if (requiredHostGroups.isEmpty()) { + requiredHostGroups.addAll(topology.getHostGroupsForComponent(component)); + } + + return requiredHostGroups; + } } /** @@ -1122,23 +1202,27 @@ public class BlueprintConfigurationProcessor { /** * Append 'm' to the original property value if it doesn't already exist. * - * - * @param hostGroups host groups - * @param origValue original value of property - * @param properties all properties - * @param stackDefinition stack used for cluster creation + * @param propertyName property name + * @param origValue original value of property + * @param properties all properties + * @param topology cluster topology * * @return property with 'm' appended */ @Override - public String updateForClusterCreate(Map<String, ? extends HostGroup> hostGroups, - String propertyName, - String origValue, Map<String, - Map<String, String>> properties, - Stack stackDefinition) { + public String updateForClusterCreate(String propertyName, + String origValue, + Map<String, Map<String, String>> properties, + ClusterTopology topology) { return origValue.endsWith("m") ? origValue : origValue + 'm'; } + + @Override + public Collection<String> getRequiredHostGroups(String origValue, + Map<String, Map<String, String>> properties, ClusterTopology topology) { + return Collections.emptySet(); + } } /** @@ -1159,21 +1243,20 @@ public class BlueprintConfigurationProcessor { /** * Return decorated form of the updated input property value. * - * @param hostGroupMap map of host group name to HostGroup - * @param origValue original value of property - * @param properties all properties - * @param stackDefinition stack used for cluster creation + * @param propertyName property name + * @param origValue original value of property + * @param properties all properties + * @param topology cluster topology * * @return Formatted output string */ @Override - public String updateForClusterCreate(Map<String, ? extends HostGroup> hostGroupMap, - String propertyName, + public String updateForClusterCreate(String propertyName, String origValue, Map<String, Map<String, String>> properties, - Stack stackDefinition) { + ClusterTopology topology) { - return doFormat(propertyUpdater.updateForClusterCreate(hostGroupMap, propertyName, origValue, properties, stackDefinition)); + return doFormat(propertyUpdater.updateForClusterCreate(propertyName, origValue, properties, topology)); } /** @@ -1184,6 +1267,13 @@ public class BlueprintConfigurationProcessor { * @return formatted output string */ public abstract String doFormat(String originalValue); + + @Override + public Collection<String> getRequiredHostGroups(String origValue, + Map<String, Map<String, String>> properties, ClusterTopology topology) { + + return propertyUpdater.getRequiredHostGroups(origValue, properties, topology); + } } /** @@ -1232,14 +1322,22 @@ public class BlueprintConfigurationProcessor { * during the Blueprint export process. */ private static class OriginalValuePropertyUpdater implements PropertyUpdater { - @Override - public String updateForClusterCreate(Map<String, ? extends HostGroup> hostGroups, - String propertyName, String origValue, + + public String updateForClusterCreate(String propertyName, + String origValue, Map<String, Map<String, String>> properties, - Stack stackDefinition) { + ClusterTopology topology) { // always return the original value, since these properties do not require update handling return origValue; } + + @Override + public Collection<String> getRequiredHostGroups(String origValue, + Map<String, Map<String, String>> properties, + ClusterTopology topology) { + + return Collections.emptySet(); + } } @@ -1264,7 +1362,11 @@ public class BlueprintConfigurationProcessor { } @Override - public String updateForClusterCreate(Map<String, ? extends HostGroup> hostGroups, String propertyName, String origValue, Map<String, Map<String, String>> properties, Stack stackDefinition) { + public String updateForClusterCreate(String propertyName, + String origValue, + Map<String, Map<String, String>> properties, + ClusterTopology topology) { + // short-circuit out any custom property values defined by the deployer if (!origValue.contains("%HOSTGROUP") && (!origValue.contains("localhost"))) { @@ -1273,8 +1375,7 @@ public class BlueprintConfigurationProcessor { return origValue; } - StringBuffer updatedResult = new StringBuffer(); - + StringBuilder updatedResult = new StringBuilder(); // split out the key/value pairs String[] keyValuePairs = origValue.split(","); boolean firstValue = true; @@ -1287,10 +1388,13 @@ public class BlueprintConfigurationProcessor { String key = keyValuePair.split("=")[0]; if (mapOfKeysToUpdaters.containsKey(key)) { - String result = mapOfKeysToUpdaters.get(key).updateForClusterCreate(hostGroups, key, keyValuePair.split("=")[1], properties, stackDefinition); + String result = mapOfKeysToUpdaters.get(key).updateForClusterCreate( + key, keyValuePair.split("=")[1], properties, topology); // append the internal property result, escape out any commas in the internal property, // this is required due to the specific syntax of templeton.hive.properties - updatedResult.append(key + "=" + result.replaceAll(",", Matcher.quoteReplacement("\\,"))); + updatedResult.append(key); + updatedResult.append("="); + updatedResult.append(result.replaceAll(",", Matcher.quoteReplacement("\\,"))); } else { updatedResult.append(keyValuePair); } @@ -1298,6 +1402,32 @@ public class BlueprintConfigurationProcessor { return updatedResult.toString(); } + + @Override + public Collection<String> getRequiredHostGroups(String origValue, + Map<String, Map<String, String>> properties, + ClusterTopology topology) { + + // short-circuit out any custom property values defined by the deployer + if (!origValue.contains("%HOSTGROUP") && + (!origValue.contains("localhost"))) { + // this property must contain FQDNs specified directly by the user + // of the Blueprint, so the processor should not attempt to update them + return Collections.emptySet(); + } + + Collection<String> requiredGroups = new HashSet<String>(); + // split out the key/value pairs + String[] keyValuePairs = origValue.split(","); + for (String keyValuePair : keyValuePairs) { + String key = keyValuePair.split("=")[0]; + if (mapOfKeysToUpdaters.containsKey(key)) { + requiredGroups.addAll(mapOfKeysToUpdaters.get(key).getRequiredHostGroups( + keyValuePair.split("=")[1], properties, topology)); + } + } + return requiredGroups; + } } /** @@ -1377,6 +1507,10 @@ public class BlueprintConfigurationProcessor { removePropertyUpdaters.put("oozie-env", oozieEnvOriginalValueMap); removePropertyUpdaters.put("oozie-site", oozieSiteOriginalValueMap); + //todo: Need to change updaters back to being static + //todo: will need to pass ClusterTopology in as necessary + + // NAMENODE hdfsSiteMap.put("dfs.http.address", new SingleHostTopologyUpdater("NAMENODE")); hdfsSiteMap.put("dfs.https.address", new SingleHostTopologyUpdater("NAMENODE")); @@ -1489,4 +1623,72 @@ public class BlueprintConfigurationProcessor { hbaseEnvMap.put("hbase_master_heapsize", new MPropertyUpdater()); hbaseEnvMap.put("hbase_regionserver_heapsize", new MPropertyUpdater()); } + + /** + * Explicitly set any properties that are required but not currently provided in the stack definition. + */ + void setMissingConfigurations(Map<String, Map<String, String>> mapClusterConfigurations) { + // AMBARI-5206 + final Map<String , String> userProps = new HashMap<String , String>(); + + Collection<String> services = clusterTopology.getBlueprint().getServices(); + // only add user properties to the map for + // services actually included in the blueprint definition + if (services.contains("OOZIE")) { + userProps.put("oozie_user", "oozie-env"); + } + + if (services.contains("HIVE")) { + userProps.put("hive_user", "hive-env"); + userProps.put("hcat_user", "hive-env"); + } + + if (services.contains("HBASE")) { + userProps.put("hbase_user", "hbase-env"); + } + + if (services.contains("FALCON")) { + userProps.put("falcon_user", "falcon-env"); + } + + + String proxyUserHosts = "hadoop.proxyuser.%s.hosts"; + String proxyUserGroups = "hadoop.proxyuser.%s.groups"; + + for (String property : userProps.keySet()) { + String configType = userProps.get(property); + Map<String, String> configs = mapClusterConfigurations.get(configType); + if (configs != null) { + String user = configs.get(property); + if (user != null && !user.isEmpty()) { + ensureProperty(mapClusterConfigurations, "core-site", String.format(proxyUserHosts, user), "*"); + ensureProperty(mapClusterConfigurations, "core-site", String.format(proxyUserGroups, user), "users"); + } + } else { + LOG.debug("setMissingConfigurations: no user configuration found for type = " + configType + + ". This may be caused by an error in the blueprint configuration."); + } + + } + } + + /** + * Ensure that the specified property exists. + * If not, set a default value. + * + * @param type config type + * @param property property name + * @param defaultValue default value + */ + private void ensureProperty(Map<String, Map<String, String>> mapClusterConfigurations, String type, String property, String defaultValue) { + Map<String, String> properties = mapClusterConfigurations.get(type); + if (properties == null) { + properties = new HashMap<String, String>(); + mapClusterConfigurations.put(type, properties); + } + + if (! properties.containsKey(property)) { + properties.put(property, defaultValue); + } + } }
