http://git-wip-us.apache.org/repos/asf/ambari/blob/65d44cd5/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackV2.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackV2.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackV2.java new file mode 100644 index 0000000..c5a6064 --- /dev/null +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackV2.java @@ -0,0 +1,829 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ambari.server.controller.internal; + +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +import org.apache.ambari.server.AmbariException; +import org.apache.ambari.server.controller.*; +import org.apache.ambari.server.orm.entities.StackEntity; + +import org.apache.ambari.server.state.AutoDeployInfo; +import org.apache.ambari.server.state.ComponentInfo; +import org.apache.ambari.server.state.DependencyInfo; +import org.apache.ambari.server.state.PropertyDependencyInfo; +import org.apache.ambari.server.state.PropertyInfo; +import org.apache.ambari.server.state.ValueAttributesInfo; + +import org.apache.ambari.server.topology.Cardinality; +import org.apache.ambari.server.topology.Configuration; + +/** + * Encapsulates stack information. + */ +public class StackV2 { + /** + * Stack name + */ + private String name; + + /** + * Stack version + */ + private String version; + + /** + * Repo version + */ + private String repoVersion; + + /** + * Map of service name to components + */ + private Map<String, Collection<String>> serviceComponents = + new HashMap<>(); + + /** + * Map of component to service + */ + private Map<String, String> componentService = new HashMap<>(); + + /** + * Map of component to dependencies + */ + private Map<String, Collection<DependencyInfo>> dependencies = + new HashMap<>(); + + /** + * Map of dependency to conditional service + */ + private Map<DependencyInfo, String> dependencyConditionalServiceMap = + new HashMap<>(); + + /** + * Map of database component name to configuration property which indicates whether + * the database in to be managed or if it is an external non-managed instance. + * If the value of the config property starts with 'New', the database is determined + * to be managed, otherwise it is non-managed. + */ + private Map<String, String> dbDependencyInfo = new HashMap<>(); + + /** + * Map of component to required cardinality + */ + private Map<String, String> cardinalityRequirements = new HashMap<>(); + + //todo: instead of all these maps from component -> * , + //todo: we should use a Component object with all of these attributes + private Set<String> masterComponents = new HashSet<>(); + + /** + * Map of component to auto-deploy information + */ + private Map<String, AutoDeployInfo> componentAutoDeployInfo = + new HashMap<>(); + + /** + * Map of service to config type properties + */ + private Map<String, Map<String, Map<String, ConfigProperty>>> serviceConfigurations = + new HashMap<>(); + + /** + * Map of service to required type properties + */ + private Map<String, Map<String, Map<String, ConfigProperty>>> requiredServiceConfigurations = + new HashMap<>(); + + /** + * Map of service to config type properties + */ + private Map<String, Map<String, ConfigProperty>> stackConfigurations = + new HashMap<>(); + + /** + * Map of service to set of excluded config types + */ + private Map<String, Set<String>> excludedConfigurationTypes = + new HashMap<>(); + + /** + * Ambari Management Controller, used to obtain Stack definitions + */ + private final AmbariManagementController controller; + + + /** + * Constructor. + * + * @param stack + * the stack (not {@code null}). + * @param ambariManagementController + * the management controller (not {@code null}). + * @throws AmbariException + */ + public StackV2(StackEntity stack, AmbariManagementController ambariManagementController) throws AmbariException { + this(stack.getStackName(), stack.getStackVersion(), ambariManagementController); + } + + /** + * Constructor. + * + * @param name stack name + * @param version stack version + * + * @throws AmbariException an exception occurred getting stack information + * for the specified name and version + */ + //todo: don't pass management controller in constructor + public StackV2(String name, String version, AmbariManagementController controller) throws AmbariException { + this.name = name; + this.version = version; + this.controller = controller; + + Set<StackServiceResponse> stackServices = controller.getStackServices( + Collections.singleton(new StackServiceRequest(name, version, null))); + + for (StackServiceResponse stackService : stackServices) { + String serviceName = stackService.getServiceName(); + parseComponents(serviceName); + parseExcludedConfigurations(stackService); + parseConfigurations(stackService); + registerConditionalDependencies(); + } + + //todo: already done for each service + parseStackConfigurations(); + } + + /** + * Obtain stack name. + * + * @return stack name + */ + public String getName() { + return name; + } + + /** + * Obtain stack version. + * + * @return stack version + */ + public String getVersion() { + return version; + } + + /** + * Obtain repo version. + * @return + */ + public String getRepoVersion() { return repoVersion; } + + Map<DependencyInfo, String> getDependencyConditionalServiceMap() { + return dependencyConditionalServiceMap; + } + + /** + * Get services contained in the stack. + * + * @return collection of all services for the stack + */ + public Collection<String> getServices() { + return serviceComponents.keySet(); + } + + /** + * Get components contained in the stack for the specified service. + * + * @param service service name + * + * @return collection of component names for the specified service + */ + public Collection<String> getComponents(String service) { + return serviceComponents.get(service); + } + + /** + * Get all service components + * + * @return map of service to associated components + */ + public Map<String, Collection<String>> getComponents() { + Map<String, Collection<String>> serviceComponents = new HashMap<>(); + for (String service : getServices()) { + Collection<String> components = new HashSet<>(); + components.addAll(getComponents(service)); + serviceComponents.put(service, components); + } + return serviceComponents; + } + + /** + * Get info for the specified component. + * + * @param component component name + * + * @return component information for the requested component + * or null if the component doesn't exist in the stack + */ + public ComponentInfo getComponentInfo(String component) { + ComponentInfo componentInfo = null; + String service = getServiceForComponent(component); + if (service != null) { + try { + componentInfo = controller.getAmbariMetaInfo().getComponent( + getName(), getVersion(), service, component); + } catch (AmbariException e) { + // just return null if component doesn't exist + } + } + return componentInfo; + } + + /** + * Get all configuration types, including excluded types for the specified service. + * + * @param service service name + * + * @return collection of all configuration types for the specified service + */ + public Collection<String> getAllConfigurationTypes(String service) { + return serviceConfigurations.get(service).keySet(); + } + + /** + * Get configuration types for the specified service. + * This doesn't include any service excluded types. + * + * @param service service name + * + * @return collection of all configuration types for the specified service + */ + public Collection<String> getConfigurationTypes(String service) { + Set<String> serviceTypes = new HashSet<>(serviceConfigurations.get(service).keySet()); + serviceTypes.removeAll(getExcludedConfigurationTypes(service)); + + return serviceTypes; + } + + /** + * Get the set of excluded configuration types for this service. + * + * @param service service name + * + * @return Set of names of excluded config types. Will not return null. + */ + public Set<String> getExcludedConfigurationTypes(String service) { + return excludedConfigurationTypes.containsKey(service) ? + excludedConfigurationTypes.get(service) : + Collections.emptySet(); + } + + /** + * Get config properties for the specified service and configuration type. + * + * @param service service name + * @param type configuration type + * + * @return map of property names to values for the specified service and configuration type + */ + public Map<String, String> getConfigurationProperties(String service, String type) { + Map<String, String> configMap = new HashMap<>(); + Map<String, ConfigProperty> configProperties = serviceConfigurations.get(service).get(type); + if (configProperties != null) { + for (Map.Entry<String, ConfigProperty> configProperty : configProperties.entrySet()) { + configMap.put(configProperty.getKey(), configProperty.getValue().getValue()); + } + } + return configMap; + } + + public Map<String, ConfigProperty> getConfigurationPropertiesWithMetadata(String service, String type) { + return serviceConfigurations.get(service).get(type); + } + + /** + * Get all required config properties for the specified service. + * + * @param service service name + * + * @return collection of all required properties for the given service + */ + public Collection<ConfigProperty> getRequiredConfigurationProperties(String service) { + Collection<ConfigProperty> requiredConfigProperties = new HashSet<>(); + Map<String, Map<String, ConfigProperty>> serviceProperties = requiredServiceConfigurations.get(service); + if (serviceProperties != null) { + for (Map.Entry<String, Map<String, ConfigProperty>> typePropertiesEntry : serviceProperties.entrySet()) { + requiredConfigProperties.addAll(typePropertiesEntry.getValue().values()); + } + } + return requiredConfigProperties; + } + + /** + * Get required config properties for the specified service which belong to the specified property type. + * + * @param service service name + * @param propertyType property type + * + * @return collection of required properties for the given service and property type + */ + public Collection<ConfigProperty> getRequiredConfigurationProperties(String service, org.apache.ambari.server.state.PropertyInfo.PropertyType propertyType) { + Collection<ConfigProperty> matchingProperties = new HashSet<>(); + Map<String, Map<String, ConfigProperty>> requiredProperties = requiredServiceConfigurations.get(service); + if (requiredProperties != null) { + for (Map.Entry<String, Map<String, ConfigProperty>> typePropertiesEntry : requiredProperties.entrySet()) { + for (ConfigProperty configProperty : typePropertiesEntry.getValue().values()) { + if (configProperty.getPropertyTypes().contains(propertyType)) { + matchingProperties.add(configProperty); + } + } + + } + } + return matchingProperties; + } + + public boolean isPasswordProperty(String service, String type, String propertyName) { + return (serviceConfigurations.containsKey(service) && + serviceConfigurations.get(service).containsKey(type) && + serviceConfigurations.get(service).get(type).containsKey(propertyName) && + serviceConfigurations.get(service).get(type).get(propertyName).getPropertyTypes(). + contains(org.apache.ambari.server.state.PropertyInfo.PropertyType.PASSWORD)); + } + + //todo + public Map<String, String> getStackConfigurationProperties(String type) { + Map<String, String> configMap = new HashMap<>(); + Map<String, ConfigProperty> configProperties = stackConfigurations.get(type); + if (configProperties != null) { + for (Map.Entry<String, ConfigProperty> configProperty : configProperties.entrySet()) { + configMap.put(configProperty.getKey(), configProperty.getValue().getValue()); + } + } + return configMap; + } + + public boolean isKerberosPrincipalNameProperty(String service, String type, String propertyName) { + return (serviceConfigurations.containsKey(service) && + serviceConfigurations.get(service).containsKey(type) && + serviceConfigurations.get(service).get(type).containsKey(propertyName) && + serviceConfigurations.get(service).get(type).get(propertyName).getPropertyTypes(). + contains(org.apache.ambari.server.state.PropertyInfo.PropertyType.KERBEROS_PRINCIPAL)); + } + /** + * Get config attributes for the specified service and configuration type. + * + * @param service service name + * @param type configuration type + * + * @return map of attribute names to map of property names to attribute values + * for the specified service and configuration type + */ + public Map<String, Map<String, String>> getConfigurationAttributes(String service, String type) { + Map<String, Map<String, String>> attributesMap = new HashMap<>(); + Map<String, ConfigProperty> configProperties = serviceConfigurations.get(service).get(type); + if (configProperties != null) { + for (Map.Entry<String, ConfigProperty> configProperty : configProperties.entrySet()) { + String propertyName = configProperty.getKey(); + Map<String, String> propertyAttributes = configProperty.getValue().getAttributes(); + if (propertyAttributes != null) { + for (Map.Entry<String, String> propertyAttribute : propertyAttributes.entrySet()) { + String attributeName = propertyAttribute.getKey(); + String attributeValue = propertyAttribute.getValue(); + if (attributeValue != null) { + Map<String, String> attributes = attributesMap.get(attributeName); + if (attributes == null) { + attributes = new HashMap<>(); + attributesMap.put(attributeName, attributes); + } + attributes.put(propertyName, attributeValue); + } + } + } + } + } + return attributesMap; + } + + //todo: + public Map<String, Map<String, String>> getStackConfigurationAttributes(String type) { + Map<String, Map<String, String>> attributesMap = new HashMap<>(); + Map<String, ConfigProperty> configProperties = stackConfigurations.get(type); + if (configProperties != null) { + for (Map.Entry<String, ConfigProperty> configProperty : configProperties.entrySet()) { + String propertyName = configProperty.getKey(); + Map<String, String> propertyAttributes = configProperty.getValue().getAttributes(); + if (propertyAttributes != null) { + for (Map.Entry<String, String> propertyAttribute : propertyAttributes.entrySet()) { + String attributeName = propertyAttribute.getKey(); + String attributeValue = propertyAttribute.getValue(); + Map<String, String> attributes = attributesMap.get(attributeName); + if (attributes == null) { + attributes = new HashMap<>(); + attributesMap.put(attributeName, attributes); + } + attributes.put(propertyName, attributeValue); + } + } + } + } + return attributesMap; + } + + /** + * Get the service for the specified component. + * + * @param component component name + * + * @return service name that contains tha specified component + */ + public String getServiceForComponent(String component) { + return componentService.get(component); + } + + /** + * Get the names of the services which contains the specified components. + * + * @param components collection of components + * + * @return collection of services which contain the specified components + */ + public Collection<String> getServicesForComponents(Collection<String> components) { + Set<String> services = new HashSet<>(); + for (String component : components) { + services.add(getServiceForComponent(component)); + } + + return services; + } + + /** + * Obtain the service name which corresponds to the specified configuration. + * + * @param config configuration type + * + * @return name of service which corresponds to the specified configuration type + */ + public String getServiceForConfigType(String config) { + for (Map.Entry<String, Map<String, Map<String, ConfigProperty>>> entry : serviceConfigurations.entrySet()) { + Map<String, Map<String, ConfigProperty>> typeMap = entry.getValue(); + String serviceName = entry.getKey(); + if (typeMap.containsKey(config) && !getExcludedConfigurationTypes(serviceName).contains(config)) { + return serviceName; + } + } + throw new IllegalArgumentException( + "Specified configuration type is not associated with any service: " + config); + } + + /** + * Return the dependencies specified for the given component. + * + * @param component component to get dependency information for + * + * @return collection of dependency information for the specified component + */ + //todo: full dependency graph + public Collection<DependencyInfo> getDependenciesForComponent(String component) { + return dependencies.containsKey(component) ? dependencies.get(component) : + Collections.emptySet(); + } + + /** + * Get the service, if any, that a component dependency is conditional on. + * + * @param dependency dependency to get conditional service for + * + * @return conditional service for provided component or null if dependency + * is not conditional on a service + */ + public String getConditionalServiceForDependency(DependencyInfo dependency) { + return dependencyConditionalServiceMap.get(dependency); + } + + public String getExternalComponentConfig(String component) { + return dbDependencyInfo.get(component); + } + + /** + * Obtain the required cardinality for the specified component. + */ + public Cardinality getCardinality(String component) { + return new Cardinality(cardinalityRequirements.get(component)); + } + + /** + * Obtain auto-deploy information for the specified component. + */ + public AutoDeployInfo getAutoDeployInfo(String component) { + return componentAutoDeployInfo.get(component); + } + + public boolean isMasterComponent(String component) { + return masterComponents.contains(component); + } + + public Configuration getConfiguration(Collection<String> services) { + Map<String, Map<String, Map<String, String>>> attributes = new HashMap<>(); + Map<String, Map<String, String>> properties = new HashMap<>(); + + for (String service : services) { + Collection<String> serviceConfigTypes = getConfigurationTypes(service); + for (String type : serviceConfigTypes) { + Map<String, String> typeProps = properties.get(type); + if (typeProps == null) { + typeProps = new HashMap<>(); + properties.put(type, typeProps); + } + typeProps.putAll(getConfigurationProperties(service, type)); + + Map<String, Map<String, String>> stackTypeAttributes = getConfigurationAttributes(service, type); + if (!stackTypeAttributes.isEmpty()) { + if (! attributes.containsKey(type)) { + attributes.put(type, new HashMap<>()); + } + Map<String, Map<String, String>> typeAttributes = attributes.get(type); + for (Map.Entry<String, Map<String, String>> attribute : stackTypeAttributes.entrySet()) { + String attributeName = attribute.getKey(); + Map<String, String> attributeProps = typeAttributes.get(attributeName); + if (attributeProps == null) { + attributeProps = new HashMap<>(); + typeAttributes.put(attributeName, attributeProps); + } + attributeProps.putAll(attribute.getValue()); + } + } + } + } + return new Configuration(properties, attributes); + } + + public Configuration getConfiguration() { + Map<String, Map<String, Map<String, String>>> stackAttributes = new HashMap<>(); + Map<String, Map<String, String>> stackConfigs = new HashMap<>(); + + for (String service : getServices()) { + for (String type : getAllConfigurationTypes(service)) { + Map<String, String> typeProps = stackConfigs.get(type); + if (typeProps == null) { + typeProps = new HashMap<>(); + stackConfigs.put(type, typeProps); + } + typeProps.putAll(getConfigurationProperties(service, type)); + + Map<String, Map<String, String>> stackTypeAttributes = getConfigurationAttributes(service, type); + if (!stackTypeAttributes.isEmpty()) { + if (! stackAttributes.containsKey(type)) { + stackAttributes.put(type, new HashMap<>()); + } + Map<String, Map<String, String>> typeAttrs = stackAttributes.get(type); + for (Map.Entry<String, Map<String, String>> attribute : stackTypeAttributes.entrySet()) { + String attributeName = attribute.getKey(); + Map<String, String> attributes = typeAttrs.get(attributeName); + if (attributes == null) { + attributes = new HashMap<>(); + typeAttrs.put(attributeName, attributes); + } + attributes.putAll(attribute.getValue()); + } + } + } + } + return new Configuration(stackConfigs, stackAttributes); + } + + /** + * Parse components for the specified service from the stack definition. + * + * @param service service name + * + * @throws AmbariException an exception occurred getting components from the stack definition + */ + private void parseComponents(String service) throws AmbariException{ + Collection<String> componentSet = new HashSet<>(); + + Set<StackServiceComponentResponse> components = controller.getStackComponents( + Collections.singleton(new StackServiceComponentRequest(name, version, service, null))); + + // stack service components + for (StackServiceComponentResponse component : components) { + String componentName = component.getComponentName(); + componentSet.add(componentName); + componentService.put(componentName, service); + String cardinality = component.getCardinality(); + if (cardinality != null) { + cardinalityRequirements.put(componentName, cardinality); + } + AutoDeployInfo autoDeploy = component.getAutoDeploy(); + if (autoDeploy != null) { + componentAutoDeployInfo.put(componentName, autoDeploy); + } + + // populate component dependencies + //todo: remove usage of AmbariMetaInfo + Collection<DependencyInfo> componentDependencies = controller.getAmbariMetaInfo().getComponentDependencies( + name, version, service, componentName); + + if (componentDependencies != null && ! componentDependencies.isEmpty()) { + dependencies.put(componentName, componentDependencies); + } + if (component.isMaster()) { + masterComponents.add(componentName); + } + } + serviceComponents.put(service, componentSet); + } + + /** + * Parse configurations for the specified service from the stack definition. + * + * @param stackService service to parse the stack configuration for + * + * @throws AmbariException an exception occurred getting configurations from the stack definition + */ + private void parseConfigurations(StackServiceResponse stackService) throws AmbariException { + String service = stackService.getServiceName(); + Map<String, Map<String, ConfigProperty>> mapServiceConfig = new HashMap<>(); + Map<String, Map<String, ConfigProperty>> mapRequiredServiceConfig = new HashMap<>(); + + + serviceConfigurations.put(service, mapServiceConfig); + requiredServiceConfigurations.put(service, mapRequiredServiceConfig); + + Set<ReadOnlyConfigurationResponse> serviceConfigs = controller.getStackConfigurations( + Collections.singleton(new StackConfigurationRequest(name, version, service, null))); + Set<ReadOnlyConfigurationResponse> stackLevelConfigs = controller.getStackLevelConfigurations( + Collections.singleton(new StackLevelConfigurationRequest(name, version, null))); + serviceConfigs.addAll(stackLevelConfigs); + + // shouldn't have any required properties in stack level configuration + for (ReadOnlyConfigurationResponse config : serviceConfigs) { + ConfigProperty configProperty = new ConfigProperty(config); + String type = configProperty.getType(); + + Map<String, ConfigProperty> mapTypeConfig = mapServiceConfig.get(type); + if (mapTypeConfig == null) { + mapTypeConfig = new HashMap<>(); + mapServiceConfig.put(type, mapTypeConfig); + } + + mapTypeConfig.put(config.getPropertyName(), configProperty); + if (config.isRequired()) { + Map<String, ConfigProperty> requiredTypeConfig = mapRequiredServiceConfig.get(type); + if (requiredTypeConfig == null) { + requiredTypeConfig = new HashMap<>(); + mapRequiredServiceConfig.put(type, requiredTypeConfig); + } + requiredTypeConfig.put(config.getPropertyName(), configProperty); + } + } + + // So far we added only config types that have properties defined + // in stack service definition. Since there might be config types + // with no properties defined we need to add those separately + Set<String> configTypes = stackService.getConfigTypes().keySet(); + for (String configType: configTypes) { + if (!mapServiceConfig.containsKey(configType)) { + mapServiceConfig.put(configType, Collections.emptyMap()); + } + } + } + + private void parseStackConfigurations () throws AmbariException { + + Set<ReadOnlyConfigurationResponse> stackLevelConfigs = controller.getStackLevelConfigurations( + Collections.singleton(new StackLevelConfigurationRequest(name, version, null))); + + for (ReadOnlyConfigurationResponse config : stackLevelConfigs) { + ConfigProperty configProperty = new ConfigProperty(config); + String type = configProperty.getType(); + + Map<String, ConfigProperty> mapTypeConfig = stackConfigurations.get(type); + if (mapTypeConfig == null) { + mapTypeConfig = new HashMap<>(); + stackConfigurations.put(type, mapTypeConfig); + } + + mapTypeConfig.put(config.getPropertyName(), + configProperty); + } + } + + /** + * Obtain the excluded configuration types from the StackServiceResponse + * + * @param stackServiceResponse the response object associated with this stack service + */ + private void parseExcludedConfigurations(StackServiceResponse stackServiceResponse) { + excludedConfigurationTypes.put(stackServiceResponse.getServiceName(), stackServiceResponse.getExcludedConfigTypes()); + } + + /** + * Register conditional dependencies. + */ + //todo: This information should be specified in the stack definition. + void registerConditionalDependencies() { + dbDependencyInfo.put("MYSQL_SERVER", "global/hive_database"); + } + + /** + * Contains a configuration property's value and attributes. + */ + public static class ConfigProperty { + private ValueAttributesInfo propertyValueAttributes = null; + private String name; + private String value; + private Map<String, String> attributes; + private Set<org.apache.ambari.server.state.PropertyInfo.PropertyType> propertyTypes; + private String type; + private Set<PropertyDependencyInfo> dependsOnProperties = + Collections.emptySet(); + + public ConfigProperty(ReadOnlyConfigurationResponse config) { + this.name = config.getPropertyName(); + this.value = config.getPropertyValue(); + this.attributes = config.getPropertyAttributes(); + this.propertyTypes = config.getPropertyType(); + this.type = normalizeType(config.getType()); + this.dependsOnProperties = config.getDependsOnProperties(); + this.propertyValueAttributes = config.getPropertyValueAttributes(); + } + + public ConfigProperty(String type, String name, String value) { + this.type = type; + this.name = name; + this.value = value; + } + + public String getName() { + return name; + } + + public String getValue() { + return value; + } + + public void setValue(String value) { + this.value = value; + } + + public String getType() { + return type; + } + + public Set<org.apache.ambari.server.state.PropertyInfo.PropertyType> getPropertyTypes() { + return propertyTypes; + } + + public void setPropertyTypes(Set<PropertyInfo.PropertyType> propertyTypes) { + this.propertyTypes = propertyTypes; + } + + public Map<String, String> getAttributes() { + return attributes; + } + + public void setAttributes(Map<String, String> attributes) { + this.attributes = attributes; + } + + Set<PropertyDependencyInfo> getDependsOnProperties() { + return this.dependsOnProperties; + } + + private String normalizeType(String type) { + //strip .xml from type + if (type.endsWith(".xml")) { + type = type.substring(0, type.length() - 4); + } + return type; + } + + public ValueAttributesInfo getPropertyValueAttributes() { + return propertyValueAttributes; + } + } +}
http://git-wip-us.apache.org/repos/asf/ambari/blob/65d44cd5/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UnitUpdater.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UnitUpdater.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UnitUpdater.java index 8b7cb67..37725b1 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UnitUpdater.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UnitUpdater.java @@ -17,15 +17,15 @@ */ package org.apache.ambari.server.controller.internal; -import static org.apache.commons.lang.StringUtils.isBlank; +import org.apache.ambari.server.topology.ClusterTopology; +import org.apache.ambari.server.topology.validators.UnitValidatedProperty; import java.util.Collection; import java.util.Collections; import java.util.Map; import java.util.Optional; -import org.apache.ambari.server.topology.ClusterTopology; -import org.apache.ambari.server.topology.validators.UnitValidatedProperty; +import static org.apache.commons.lang.StringUtils.isBlank; /** * I append the stack defined unit to the original property value. @@ -48,8 +48,9 @@ public class UnitUpdater implements BlueprintConfigurationProcessor.PropertyUpda public String updateForClusterCreate(String propertyName, String origValue, Map<String, Map<String, String>> properties, - ClusterTopology topology) { - PropertyUnit stackUnit = PropertyUnit.of(topology.getBlueprint().getStack(), serviceName, configType, propertyName); + ClusterTopology topology, + ConfigurationContext configurationContext) { + PropertyUnit stackUnit = PropertyUnit.of(configurationContext.getStack(), serviceName, configType, propertyName); PropertyValue value = PropertyValue.of(propertyName, origValue); if (value.hasUnit(stackUnit)) { return value.toString(); @@ -61,7 +62,8 @@ public class UnitUpdater implements BlueprintConfigurationProcessor.PropertyUpda } @Override - public Collection<String> getRequiredHostGroups(String propertyName, String origValue, Map<String, Map<String, String>> properties, ClusterTopology topology) { + public Collection<String> getRequiredHostGroups(String propertyName, String origValue, Map<String, Map<String, String>> properties, + ClusterTopology topology, ConfigurationContext configurationContext) { return Collections.emptySet(); } @@ -69,18 +71,18 @@ public class UnitUpdater implements BlueprintConfigurationProcessor.PropertyUpda private static final String DEFAULT_UNIT = "m"; private final String unit; - public static PropertyUnit of(Stack stack, UnitValidatedProperty property) { + public static PropertyUnit of(StackV2 stack, UnitValidatedProperty property) { return PropertyUnit.of(stack, property.getServiceName(), property.getConfigType(), property.getPropertyName()); } - public static PropertyUnit of(Stack stack, String serviceName, String configType, String propertyName) { + public static PropertyUnit of(StackV2 stack, String serviceName, String configType, String propertyName) { return new PropertyUnit( stackUnit(stack, serviceName, configType, propertyName) .map(PropertyUnit::toJvmUnit) .orElse(DEFAULT_UNIT)); } - private static Optional<String> stackUnit(Stack stack, String serviceName, String configType, String propertyName) { + private static Optional<String> stackUnit(StackV2 stack, String serviceName, String configType, String propertyName) { try { return Optional.ofNullable( stack.getConfigurationPropertiesWithMetadata(serviceName, configType) http://git-wip-us.apache.org/repos/asf/ambari/blob/65d44cd5/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java index 5af2a86..3a65662 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java @@ -18,82 +18,36 @@ package org.apache.ambari.server.topology; -import java.util.Collection; -import java.util.Collections; -import java.util.Comparator; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.SortedSet; -import java.util.TreeSet; -import java.util.concurrent.Callable; -import java.util.concurrent.atomic.AtomicLong; -import java.util.concurrent.locks.Lock; - -import javax.annotation.Nullable; -import javax.inject.Inject; - -import org.apache.ambari.server.AmbariException; -import org.apache.ambari.server.ClusterNotFoundException; -import org.apache.ambari.server.DuplicateResourceException; -import org.apache.ambari.server.Role; -import org.apache.ambari.server.RoleCommand; +import com.google.common.collect.Iterables; +import com.google.common.collect.Sets; +import com.google.common.util.concurrent.Striped; +import com.google.inject.Provider; +import org.apache.ambari.server.*; import org.apache.ambari.server.actionmanager.HostRoleCommand; import org.apache.ambari.server.actionmanager.HostRoleCommandFactory; import org.apache.ambari.server.actionmanager.HostRoleStatus; -import org.apache.ambari.server.controller.AmbariManagementController; -import org.apache.ambari.server.controller.AmbariServer; -import org.apache.ambari.server.controller.ClusterRequest; -import org.apache.ambari.server.controller.ConfigGroupRequest; -import org.apache.ambari.server.controller.ConfigurationRequest; -import org.apache.ambari.server.controller.RequestStatusResponse; -import org.apache.ambari.server.controller.ServiceComponentHostRequest; -import org.apache.ambari.server.controller.ServiceComponentRequest; -import org.apache.ambari.server.controller.ServiceRequest; -import org.apache.ambari.server.controller.internal.AbstractResourceProvider; -import org.apache.ambari.server.controller.internal.ComponentResourceProvider; -import org.apache.ambari.server.controller.internal.ConfigGroupResourceProvider; -import org.apache.ambari.server.controller.internal.HostComponentResourceProvider; -import org.apache.ambari.server.controller.internal.HostResourceProvider; -import org.apache.ambari.server.controller.internal.ProvisionClusterRequest; -import org.apache.ambari.server.controller.internal.RequestImpl; -import org.apache.ambari.server.controller.internal.ServiceResourceProvider; -import org.apache.ambari.server.controller.internal.Stack; -import org.apache.ambari.server.controller.internal.VersionDefinitionResourceProvider; +import org.apache.ambari.server.controller.*; +import org.apache.ambari.server.controller.internal.*; import org.apache.ambari.server.controller.predicate.EqualsPredicate; import org.apache.ambari.server.controller.spi.ClusterController; import org.apache.ambari.server.controller.spi.Predicate; -import org.apache.ambari.server.controller.spi.Request; -import org.apache.ambari.server.controller.spi.RequestStatus; import org.apache.ambari.server.controller.spi.Resource; import org.apache.ambari.server.controller.utilities.ClusterControllerHelper; import org.apache.ambari.server.orm.dao.RepositoryVersionDAO; import org.apache.ambari.server.orm.entities.RepositoryVersionEntity; import org.apache.ambari.server.security.authorization.AuthorizationException; -import org.apache.ambari.server.state.Cluster; -import org.apache.ambari.server.state.Clusters; -import org.apache.ambari.server.state.Config; -import org.apache.ambari.server.state.ConfigFactory; -import org.apache.ambari.server.state.ConfigHelper; -import org.apache.ambari.server.state.DesiredConfig; -import org.apache.ambari.server.state.Host; -import org.apache.ambari.server.state.RepositoryType; -import org.apache.ambari.server.state.SecurityType; -import org.apache.ambari.server.state.StackId; +import org.apache.ambari.server.state.*; import org.apache.ambari.server.state.configgroup.ConfigGroup; import org.apache.ambari.server.utils.RetryHelper; -import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.common.base.Function; -import com.google.common.collect.Collections2; -import com.google.common.collect.Iterables; -import com.google.common.collect.Sets; -import com.google.common.util.concurrent.Striped; -import com.google.inject.Provider; +import javax.annotation.Nullable; +import javax.inject.Inject; +import java.util.*; +import java.util.concurrent.Callable; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.locks.Lock; /** @@ -130,6 +84,7 @@ public class AmbariContext { private static HostRoleCommandFactory hostRoleCommandFactory; private static HostResourceProvider hostResourceProvider; + private static ServiceGroupResourceProvider serviceGroupResourceProvider; private static ServiceResourceProvider serviceResourceProvider; private static ComponentResourceProvider componentResourceProvider; private static HostComponentResourceProvider hostComponentResourceProvider; @@ -200,95 +155,12 @@ public class AmbariContext { return getController().getActionManager().getTasks(ids); } - public void createAmbariResources(ClusterTopology topology, String clusterName, SecurityType securityType, - String repoVersionString, Long repoVersionId) { - Stack stack = topology.getBlueprint().getStack(); - StackId stackId = new StackId(stack.getName(), stack.getVersion()); - - RepositoryVersionEntity repoVersion = null; - if (StringUtils.isEmpty(repoVersionString) && null == repoVersionId) { - List<RepositoryVersionEntity> stackRepoVersions = repositoryVersionDAO.findByStack(stackId); - - if (stackRepoVersions.isEmpty()) { - // !!! no repos, try to get the version for the stack - VersionDefinitionResourceProvider vdfProvider = getVersionDefinitionResourceProvider(); - - Map<String, Object> properties = new HashMap<>(); - properties.put(VersionDefinitionResourceProvider.VERSION_DEF_AVAILABLE_DEFINITION, stackId.toString()); - - Request request = new RequestImpl(Collections.<String>emptySet(), - Collections.singleton(properties), Collections.<String, String>emptyMap(), null); - - Long defaultRepoVersionId = null; - - try { - RequestStatus requestStatus = vdfProvider.createResources(request); - if (!requestStatus.getAssociatedResources().isEmpty()) { - Resource resource = requestStatus.getAssociatedResources().iterator().next(); - defaultRepoVersionId = (Long) resource.getPropertyValue(VersionDefinitionResourceProvider.VERSION_DEF_ID); - } - } catch (Exception e) { - throw new IllegalArgumentException(String.format( - "Failed to create a default repository version definition for stack %s. " - + "This typically is a result of not loading the stack correctly or being able " - + "to load information about released versions. Create a repository version " - + " and try again.", stackId), e); - } - - repoVersion = repositoryVersionDAO.findByPK(defaultRepoVersionId); - // !!! better not! - if (null == repoVersion) { - throw new IllegalArgumentException(String.format( - "Failed to load the default repository version definition for stack %s. " - + "Check for a valid repository version and try again.", stackId)); - } - - } else if (stackRepoVersions.size() > 1) { - - Function<RepositoryVersionEntity, String> function = new Function<RepositoryVersionEntity, String>() { - @Override - public String apply(RepositoryVersionEntity input) { - return input.getVersion(); - } - }; - - Collection<String> versions = Collections2.transform(stackRepoVersions, function); - - throw new IllegalArgumentException(String.format("Several repositories were found for %s: %s. Specify the version" - + " with '%s'", stackId, StringUtils.join(versions, ", "), ProvisionClusterRequest.REPO_VERSION_PROPERTY)); - } else { - repoVersion = stackRepoVersions.get(0); - LOG.warn("Cluster is being provisioned using the single matching repository version {}", repoVersion.getVersion()); - } - } else if (null != repoVersionId){ - repoVersion = repositoryVersionDAO.findByPK(repoVersionId); - - if (null == repoVersion) { - throw new IllegalArgumentException(String.format( - "Could not identify repository version with repository version id %s for installing services. " - + "Specify a valid repository version id with '%s'", - repoVersionId, ProvisionClusterRequest.REPO_VERSION_ID_PROPERTY)); - } - } else { - repoVersion = repositoryVersionDAO.findByStackAndVersion(stackId, repoVersionString); - - if (null == repoVersion) { - throw new IllegalArgumentException(String.format( - "Could not identify repository version with stack %s and version %s for installing services. " - + "Specify a valid version with '%s'", - stackId, repoVersionString, ProvisionClusterRequest.REPO_VERSION_PROPERTY)); - } - } + public void createAmbariResources(ClusterTopology topology, String clusterName, SecurityType securityType) { - // only use a STANDARD repo when creating a new cluster - if (repoVersion.getType() != RepositoryType.STANDARD) { - throw new IllegalArgumentException(String.format( - "Unable to create a cluster using the following repository since it is not a STANDARD type: %s", - repoVersion)); - } + StackV2 stack = topology.getBlueprint().getStacks().iterator().next(); createAmbariClusterResource(clusterName, stack.getName(), stack.getVersion(), securityType); - createAmbariServiceAndComponentResources(topology, clusterName, stackId, repoVersion.getId()); + createAmbariServiceAndComponentResources(topology, clusterName); } public void createAmbariClusterResource(String clusterName, String stackName, String stackVersion, SecurityType securityType) { @@ -314,34 +186,56 @@ public class AmbariContext { } } - public void createAmbariServiceAndComponentResources(ClusterTopology topology, String clusterName, - StackId stackId, Long repositoryVersionId) { - Collection<String> services = topology.getBlueprint().getServices(); + public void createAmbariServiceAndComponentResources(ClusterTopology topology, String clusterName) { - try { - Cluster cluster = getController().getClusters().getCluster(clusterName); - services.removeAll(cluster.getServices().keySet()); - } catch (AmbariException e) { - throw new RuntimeException("Failed to persist service and component resources: " + e, e); - } + Collection<ServiceGroup> serviceGroups = topology.getBlueprint().getServiceGroups(); + Set<ServiceGroupRequest> serviceGroupRequests = new HashSet<>(); Set<ServiceRequest> serviceRequests = new HashSet<>(); Set<ServiceComponentRequest> componentRequests = new HashSet<>(); - for (String service : services) { - String credentialStoreEnabled = topology.getBlueprint().getCredentialStoreEnabled(service); - serviceRequests.add(new ServiceRequest(clusterName, null, service, service, - repositoryVersionId, null, credentialStoreEnabled, null)); - - for (String component : topology.getBlueprint().getComponents(service)) { - String recoveryEnabled = topology.getBlueprint().getRecoveryEnabled(service, component); - componentRequests.add(new ServiceComponentRequest(clusterName, service, component, null, recoveryEnabled)); + + for (ServiceGroup serviceGroup : serviceGroups) { + serviceGroupRequests.add(new ServiceGroupRequest(clusterName, serviceGroup.getName())); + + for (Service service : serviceGroup.getServices()) { + String credentialStoreEnabled = topology.getBlueprint().getCredentialStoreEnabled(service.getType()); + + StackV2 stack = service.getStack(); + StackId stackId = new StackId(stack.getName(), stack.getVersion()); + RepositoryVersionEntity repoVersion = repositoryVersionDAO.findByStackAndVersion(stackId, stack.getRepoVersion()); + + if (null == repoVersion) { + throw new IllegalArgumentException(String.format( + "Could not identify repository version with stack %s and version %s for installing services. " + + "Specify a valid version with '%s'", + stackId, stack.getRepoVersion(), ProvisionClusterRequest.REPO_VERSION_PROPERTY)); + } + + // only use a STANDARD repo when creating a new cluster + if (repoVersion.getType() != RepositoryType.STANDARD) { + throw new IllegalArgumentException(String.format( + "Unable to create a cluster using the following repository since it is not a STANDARD type: %s", + repoVersion)); + } + + serviceRequests.add(new ServiceRequest(clusterName, serviceGroup.getName(), service.getType(), service.getName(), + repoVersion.getId(), null, credentialStoreEnabled, null)); + + for (ComponentV2 component : topology.getBlueprint().getComponents(service)) { + String recoveryEnabled = topology.getBlueprint().getRecoveryEnabled(component); + componentRequests.add(new ServiceComponentRequest(clusterName, serviceGroup.getName(), service.getName(), + component.getName(), null, recoveryEnabled)); + } } + } try { + getServiceGroupResourceProvider().createServiceGroups(serviceGroupRequests); getServiceResourceProvider().createServices(serviceRequests); getComponentResourceProvider().createComponents(componentRequests); } catch (AmbariException | AuthorizationException e) { throw new RuntimeException("Failed to persist service and component resources: " + e, e); } + // set all services state to INSTALLED->STARTED // this is required so the user can start failed services at the service level Map<String, Object> installProps = new HashMap<>(); @@ -351,20 +245,20 @@ public class AmbariContext { startProps.put(ServiceResourceProvider.SERVICE_SERVICE_STATE_PROPERTY_ID, "STARTED"); startProps.put(ServiceResourceProvider.SERVICE_CLUSTER_NAME_PROPERTY_ID, clusterName); Predicate predicate = new EqualsPredicate<>( - ServiceResourceProvider.SERVICE_CLUSTER_NAME_PROPERTY_ID, clusterName); + ServiceResourceProvider.SERVICE_CLUSTER_NAME_PROPERTY_ID, clusterName); try { getServiceResourceProvider().updateResources( - new RequestImpl(null, Collections.singleton(installProps), null, null), predicate); + new RequestImpl(null, Collections.singleton(installProps), null, null), predicate); getServiceResourceProvider().updateResources( - new RequestImpl(null, Collections.singleton(startProps), null, null), predicate); + new RequestImpl(null, Collections.singleton(startProps), null, null), predicate); } catch (Exception e) { // just log as this won't prevent cluster from being provisioned correctly LOG.error("Unable to update state of services during cluster provision: " + e, e); } } - public void createAmbariHostResources(long clusterId, String hostName, Map<String, Collection<String>> components) { + public void createAmbariHostResources(long clusterId, String hostName, Map<Service, Collection<ComponentV2>> components) { Host host; try { host = getController().getClusters().getHost(hostName); @@ -398,13 +292,14 @@ public class AmbariContext { final Set<ServiceComponentHostRequest> requests = new HashSet<>(); - for (Map.Entry<String, Collection<String>> entry : components.entrySet()) { - String service = entry.getKey(); - for (String component : entry.getValue()) { + for (Map.Entry<Service, Collection<ComponentV2>> entry : components.entrySet()) { + Service service = entry.getKey(); + for (ComponentV2 component : entry.getValue()) { //todo: handle this in a generic manner. These checks are all over the code try { - if (cluster.getService(service) != null && !component.equals("AMBARI_SERVER")) { - requests.add(new ServiceComponentHostRequest(clusterName, null, service, component, hostName, null)); + if (cluster.getService(service.getName()) != null && !component.getType().equals("AMBARI_SERVER")) { + requests.add(new ServiceComponentHostRequest(clusterName, service.getServiceGroup().getName(), + service.getName(), component.getName(), hostName, null)); } } catch(AmbariException se) { LOG.warn("Service already deleted from cluster: {}", service); @@ -718,34 +613,32 @@ public class AmbariContext { * and the hosts associated with the host group are assigned to the config group. */ private void createConfigGroupsAndRegisterHost(ClusterTopology topology, String groupName) throws AmbariException { - Map<String, Map<String, Config>> groupConfigs = new HashMap<>(); - Stack stack = topology.getBlueprint().getStack(); - - // get the host-group config with cluster creation template overrides - Configuration topologyHostGroupConfig = topology. - getHostGroupInfo().get(groupName).getConfiguration(); - - // only get user provided configuration for host group which includes only CCT/HG and BP/HG properties - Map<String, Map<String, String>> userProvidedGroupProperties = - topologyHostGroupConfig.getFullProperties(1); - - // iterate over topo host group configs which were defined in - for (Map.Entry<String, Map<String, String>> entry : userProvidedGroupProperties.entrySet()) { - String type = entry.getKey(); - String service = stack.getServiceForConfigType(type); - Config config = configFactory.createReadOnly(type, groupName, entry.getValue(), null); - //todo: attributes - Map<String, Config> serviceConfigs = groupConfigs.get(service); - if (serviceConfigs == null) { - serviceConfigs = new HashMap<>(); - groupConfigs.put(service, serviceConfigs); + + Map<Service, Map<String, Config>> groupConfigs = new HashMap<>(); + + + // only get user provided configuration for host group per service which includes only CCT/HG and BP/HG properties + Collection<Service> serviceConfigurations = topology.getHostGroupInfo().get(groupName).getServiceConfigs(); + serviceConfigurations.forEach(service -> { + Map<String, Map<String, String>> userProvidedGroupProperties = service.getConfiguration().getProperties(); + + // iterate over topo host group configs which were defined in + for (Map.Entry<String, Map<String, String>> entry : userProvidedGroupProperties.entrySet()) { + String type = entry.getKey(); + Config config = configFactory.createReadOnly(type, groupName, entry.getValue(), null); + //todo: attributes + Map<String, Config> serviceConfigs = groupConfigs.get(service); + if (serviceConfigs == null) { + serviceConfigs = new HashMap<>(); + groupConfigs.put(service, serviceConfigs); + } + serviceConfigs.put(type, config); } - serviceConfigs.put(type, config); - } + }); String bpName = topology.getBlueprint().getName(); - for (Map.Entry<String, Map<String, Config>> entry : groupConfigs.entrySet()) { - String service = entry.getKey(); + for (Map.Entry<Service, Map<String, Config>> entry : groupConfigs.entrySet()) { + Service service = entry.getKey(); Map<String, Config> serviceConfigs = entry.getValue(); String absoluteGroupName = getConfigurationGroupName(bpName, groupName); Collection<String> groupHosts; @@ -771,7 +664,7 @@ public class AmbariContext { }); ConfigGroupRequest request = new ConfigGroupRequest(null, clusterName, - absoluteGroupName, service, service, "Host Group Configuration", + absoluteGroupName, service.getName(), service.getName(), "Host Group Configuration", Sets.newHashSet(filteredGroupHosts), serviceConfigs); // get the config group provider and create config group resource @@ -826,6 +719,14 @@ public class AmbariContext { return hostComponentResourceProvider; } + private synchronized ServiceGroupResourceProvider getServiceGroupResourceProvider() { + if (serviceGroupResourceProvider == null) { + serviceGroupResourceProvider = (ServiceGroupResourceProvider) ClusterControllerHelper. + getClusterController().ensureResourceProvider(Resource.Type.ServiceGroup); + } + return serviceGroupResourceProvider; + } + private synchronized ServiceResourceProvider getServiceResourceProvider() { if (serviceResourceProvider == null) { serviceResourceProvider = (ServiceResourceProvider) ClusterControllerHelper. http://git-wip-us.apache.org/repos/asf/ambari/blob/65d44cd5/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintFactory.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintFactory.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintFactory.java index 404068d..c8860f6 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintFactory.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintFactory.java @@ -19,13 +19,7 @@ package org.apache.ambari.server.topology; -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Map; -import java.util.Set; - +import com.google.inject.Inject; import org.apache.ambari.server.AmbariException; import org.apache.ambari.server.ObjectNotFoundException; import org.apache.ambari.server.controller.AmbariManagementController; @@ -37,7 +31,7 @@ import org.apache.ambari.server.orm.dao.BlueprintDAO; import org.apache.ambari.server.orm.entities.BlueprintEntity; import org.apache.ambari.server.stack.NoSuchStackException; -import com.google.inject.Inject; +import java.util.*; /** * Create a Blueprint instance. @@ -82,10 +76,10 @@ public class BlueprintFactory { this.stackFactory = stackFactory; } - public Blueprint getBlueprint(String blueprintName) throws NoSuchStackException { + public BlueprintV2 getBlueprint(String blueprintName) throws NoSuchStackException { BlueprintEntity entity = blueprintDAO.findByName(blueprintName); //todo: just return null? - return entity == null ? null : new BlueprintImpl(entity); + return entity == null ? null : new BlueprintImplV2(entity); } /** http://git-wip-us.apache.org/repos/asf/ambari/blob/65d44cd5/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintImplV2.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintImplV2.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintImplV2.java index 9bde795..79456e5 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintImplV2.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintImplV2.java @@ -19,18 +19,20 @@ package org.apache.ambari.server.topology; +import org.apache.ambari.server.controller.internal.StackV2; +import org.apache.ambari.server.orm.entities.BlueprintEntity; + import java.util.Collection; import java.util.List; import java.util.Map; - -import org.apache.ambari.server.controller.StackV2; -import org.apache.ambari.server.orm.entities.BlueprintEntity; +import java.util.stream.Collectors; /** * Blueprint implementation. */ public class BlueprintImplV2 implements BlueprintV2 { - + public BlueprintImplV2(BlueprintEntity e) { + } @Override public String getName() { @@ -63,11 +65,38 @@ public class BlueprintImplV2 implements BlueprintV2 { } @Override + public Collection<String> getAllServiceTypes() { + return null; + } + + @Override + public Collection<Service> getServicesByType(String serviceType) { + return getAllServices().stream().filter( + service -> service.getType().equalsIgnoreCase(serviceType)).collect(Collectors.toList()); + } + + @Override + public Collection<Service> getServicesFromServiceGroup(ServiceGroup serviceGroup, String serviceType) { + if (serviceType == null) { + return serviceGroup.getServices(); + } else { + return serviceGroup.getServices().stream().filter( + service -> service.getType().equalsIgnoreCase(serviceType)).collect(Collectors.toList()); + } + } + + @Override public Collection<ComponentV2> getComponents(Service service) { return null; } @Override + public Collection<ComponentV2> getComponentsByType(Service service, String componentType) { + return getComponents(service).stream().filter( + compnoent -> compnoent.getType().equalsIgnoreCase(componentType)).collect(Collectors.toList()); + } + + @Override public Collection<HostGroupV2> getHostGroupsForService(Service service) { return null; } @@ -78,6 +107,7 @@ public class BlueprintImplV2 implements BlueprintV2 { } @Override + @Deprecated public Configuration getConfiguration() { return null; } @@ -88,7 +118,7 @@ public class BlueprintImplV2 implements BlueprintV2 { } @Override - public String getRecoveryEnabled(String serviceName, String componentName) { + public String getRecoveryEnabled(ComponentV2 component) { return null; } http://git-wip-us.apache.org/repos/asf/ambari/blob/65d44cd5/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintV2.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintV2.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintV2.java index 1fea966..3c71e41 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintV2.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintV2.java @@ -18,12 +18,13 @@ package org.apache.ambari.server.topology; +import org.apache.ambari.server.controller.internal.StackV2; +import org.apache.ambari.server.orm.entities.BlueprintEntity; + import java.util.Collection; import java.util.List; import java.util.Map; -import org.apache.ambari.server.controller.StackV2; -import org.apache.ambari.server.orm.entities.BlueprintEntity; /** * Blueprint representation. @@ -69,6 +70,29 @@ public interface BlueprintV2 { */ Collection<Service> getAllServices(); + + /** + * Get all of the service types represented in the blueprint. + * + * @return collection of all represented service types + */ + Collection<String> getAllServiceTypes(); + + /** + * Get all of the services represented in the blueprint with a given type. + * + * @return collection of all represented services represented in the blueprint with a given type. + */ + Collection<Service> getServicesByType(String serviceType); + + /** + * Get services by type from a service group. + * @param serviceGroup + * @param serviceType + * @return + */ + Collection<Service> getServicesFromServiceGroup(ServiceGroup serviceGroup, String serviceType); + /** * Get the components that are included in the blueprint for the specified service. * @@ -78,6 +102,14 @@ public interface BlueprintV2 { */ Collection<ComponentV2> getComponents(Service service); + /** + * Get components by type from a service. + * @param service + * @param componentType + * @return + */ + Collection<ComponentV2> getComponentsByType(Service service, String componentType); + /** * Get the host groups which contain components for the specified service. @@ -106,6 +138,7 @@ public interface BlueprintV2 { * * @return blueprint cluster scoped configuration */ + @Deprecated Configuration getConfiguration(); /** @@ -121,12 +154,11 @@ public interface BlueprintV2 { /** * Get whether a component is enabled for auto start. * - * @param serviceName - Service name. - * @param componentName - Component name. + * @param component - Component. * * @return null if value is not specified; true or false if specified. */ - String getRecoveryEnabled(String serviceName, String componentName); + String getRecoveryEnabled(ComponentV2 component); /** * Get whether a service is enabled for credential store use. http://git-wip-us.apache.org/repos/asf/ambari/blob/65d44cd5/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintValidatorImpl.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintValidatorImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintValidatorImpl.java index 1a43b85..0244d7b 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintValidatorImpl.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintValidatorImpl.java @@ -18,14 +18,6 @@ package org.apache.ambari.server.topology; -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.Map; -import java.util.Set; - import org.apache.ambari.server.controller.internal.Stack; import org.apache.ambari.server.state.AutoDeployInfo; import org.apache.ambari.server.state.DependencyConditionInfo; @@ -35,6 +27,11 @@ import org.apache.ambari.server.utils.VersionUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; + /** * Default blueprint validator. */ @@ -85,6 +82,8 @@ public class BlueprintValidatorImpl implements BlueprintValidator { @Override public void validateRequiredProperties() throws InvalidTopologyException { + //TODO + // ConfigurationContext configurationContext = new ConfigurationContext(blueprint.getStacks().iterator().next(), blueprint.getConfiguration()); // we don't want to include default stack properties so we can't just use hostGroup full properties Map<String, Map<String, String>> clusterConfigurations = blueprint.getConfiguration().getProperties(); @@ -131,31 +130,31 @@ public class BlueprintValidatorImpl implements BlueprintValidator { " using existing db!"); } } - if (ClusterTopologyImpl.isNameNodeHAEnabled(clusterConfigurations) && component.equals("NAMENODE")) { - Map<String, String> hadoopEnvConfig = clusterConfigurations.get("hadoop-env"); - if(hadoopEnvConfig != null && !hadoopEnvConfig.isEmpty() && hadoopEnvConfig.containsKey("dfs_ha_initial_namenode_active") && hadoopEnvConfig.containsKey("dfs_ha_initial_namenode_standby")) { - ArrayList<HostGroup> hostGroupsForComponent = new ArrayList<>(blueprint.getHostGroupsForComponent(component)); - Set<String> givenHostGroups = new HashSet<>(); - givenHostGroups.add(hadoopEnvConfig.get("dfs_ha_initial_namenode_active")); - givenHostGroups.add(hadoopEnvConfig.get("dfs_ha_initial_namenode_standby")); - if(givenHostGroups.size() != hostGroupsForComponent.size()) { - throw new IllegalArgumentException("NAMENODE HA host groups mapped incorrectly for properties 'dfs_ha_initial_namenode_active' and 'dfs_ha_initial_namenode_standby'. Expected Host groups are :" + hostGroupsForComponent); - } - if(HostGroup.HOSTGROUP_REGEX.matcher(hadoopEnvConfig.get("dfs_ha_initial_namenode_active")).matches() && HostGroup.HOSTGROUP_REGEX.matcher(hadoopEnvConfig.get("dfs_ha_initial_namenode_standby")).matches()){ - for (HostGroup hostGroupForComponent : hostGroupsForComponent) { - Iterator<String> itr = givenHostGroups.iterator(); - while(itr.hasNext()){ - if(itr.next().contains(hostGroupForComponent.getName())){ - itr.remove(); - } - } - } - if(!givenHostGroups.isEmpty()){ - throw new IllegalArgumentException("NAMENODE HA host groups mapped incorrectly for properties 'dfs_ha_initial_namenode_active' and 'dfs_ha_initial_namenode_standby'. Expected Host groups are :" + hostGroupsForComponent); - } - } - } - } +// if (configurationContext.isNameNodeHAEnabled(clusterConfigurations) && component.equals("NAMENODE")) { +// Map<String, String> hadoopEnvConfig = clusterConfigurations.get("hadoop-env"); +// if(hadoopEnvConfig != null && !hadoopEnvConfig.isEmpty() && hadoopEnvConfig.containsKey("dfs_ha_initial_namenode_active") && hadoopEnvConfig.containsKey("dfs_ha_initial_namenode_standby")) { +// ArrayList<HostGroup> hostGroupsForComponent = new ArrayList<>(blueprint.getHostGroupsForComponent(component)); +// Set<String> givenHostGroups = new HashSet<>(); +// givenHostGroups.add(hadoopEnvConfig.get("dfs_ha_initial_namenode_active")); +// givenHostGroups.add(hadoopEnvConfig.get("dfs_ha_initial_namenode_standby")); +// if(givenHostGroups.size() != hostGroupsForComponent.size()) { +// throw new IllegalArgumentException("NAMENODE HA host groups mapped incorrectly for properties 'dfs_ha_initial_namenode_active' and 'dfs_ha_initial_namenode_standby'. Expected Host groups are :" + hostGroupsForComponent); +// } +// if(HostGroup.HOSTGROUP_REGEX.matcher(hadoopEnvConfig.get("dfs_ha_initial_namenode_active")).matches() && HostGroup.HOSTGROUP_REGEX.matcher(hadoopEnvConfig.get("dfs_ha_initial_namenode_standby")).matches()){ +// for (HostGroup hostGroupForComponent : hostGroupsForComponent) { +// Iterator<String> itr = givenHostGroups.iterator(); +// while(itr.hasNext()){ +// if(itr.next().contains(hostGroupForComponent.getName())){ +// itr.remove(); +// } +// } +// } +// if(!givenHostGroups.isEmpty()){ +// throw new IllegalArgumentException("NAMENODE HA host groups mapped incorrectly for properties 'dfs_ha_initial_namenode_active' and 'dfs_ha_initial_namenode_standby'. Expected Host groups are :" + hostGroupsForComponent); +// } +// } +// } +// } if (component.equals("HIVE_METASTORE")) { Map<String, String> hiveEnvConfig = clusterConfigurations.get("hive-env"); @@ -293,12 +292,13 @@ public class BlueprintValidatorImpl implements BlueprintValidator { Map<String, Map<String, String>> configProperties = blueprint.getConfiguration().getProperties(); Collection<String> cardinalityFailures = new HashSet<>(); //todo: don't hard code this HA logic here - if (ClusterTopologyImpl.isNameNodeHAEnabled(configProperties) && - (component.equals("SECONDARY_NAMENODE"))) { - // override the cardinality for this component in an HA deployment, - // since the SECONDARY_NAMENODE should not be started in this scenario - cardinality = new Cardinality("0"); - } +//TODO +// if (ClusterTopologyImpl.isNameNodeHAEnabled(configProperties) && +// (component.equals("SECONDARY_NAMENODE"))) { +// // override the cardinality for this component in an HA deployment, +// // since the SECONDARY_NAMENODE should not be started in this scenario +// cardinality = new Cardinality("0"); +// } int actualCount = blueprint.getHostGroupsForComponent(component).size(); if (! cardinality.isValidCount(actualCount)) { http://git-wip-us.apache.org/repos/asf/ambari/blob/65d44cd5/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterConfigurationRequest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterConfigurationRequest.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterConfigurationRequest.java index 7bd377f..48f8fec 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterConfigurationRequest.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterConfigurationRequest.java @@ -18,25 +18,11 @@ package org.apache.ambari.server.topology; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - import org.apache.ambari.server.AmbariException; import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorBlueprintProcessor; import org.apache.ambari.server.controller.ClusterRequest; import org.apache.ambari.server.controller.ConfigurationRequest; -import org.apache.ambari.server.controller.internal.BlueprintConfigurationProcessor; -import org.apache.ambari.server.controller.internal.ClusterResourceProvider; -import org.apache.ambari.server.controller.internal.ConfigurationTopologyException; -import org.apache.ambari.server.controller.internal.Stack; +import org.apache.ambari.server.controller.internal.*; import org.apache.ambari.server.serveraction.kerberos.KerberosInvalidConfigurationException; import org.apache.ambari.server.state.Cluster; import org.apache.ambari.server.state.SecurityType; @@ -46,6 +32,10 @@ import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.*; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + /** * Responsible for cluster configuration. */ @@ -63,7 +53,6 @@ public class ClusterConfigurationRequest { private ClusterTopology clusterTopology; private BlueprintConfigurationProcessor configurationProcessor; private StackAdvisorBlueprintProcessor stackAdvisorBlueprintProcessor; - private Stack stack; private boolean configureSecurity = false; public ClusterConfigurationRequest(AmbariContext ambariContext, ClusterTopology topology, boolean setInitial, StackAdvisorBlueprintProcessor stackAdvisorBlueprintProcessor, boolean configureSecurity) { @@ -75,10 +64,11 @@ public class ClusterConfigurationRequest { StackAdvisorBlueprintProcessor stackAdvisorBlueprintProcessor) { this.ambariContext = ambariContext; this.clusterTopology = clusterTopology; - Blueprint blueprint = clusterTopology.getBlueprint(); - this.stack = blueprint.getStack(); + BlueprintV2 blueprint = clusterTopology.getBlueprint(); // set initial configuration (not topology resolved) - this.configurationProcessor = new BlueprintConfigurationProcessor(clusterTopology); + //TODO set up proper ConfigurationContext + ConfigurationContext configurationContext = new ConfigurationContext(blueprint.getStacks().iterator().next(), blueprint.getConfiguration()); + this.configurationProcessor = new BlueprintConfigurationProcessor(clusterTopology, configurationContext); this.stackAdvisorBlueprintProcessor = stackAdvisorBlueprintProcessor; removeOrphanConfigTypes(); if (setInitial) { @@ -90,7 +80,7 @@ public class ClusterConfigurationRequest { * Remove config-types from the given configuration if there is no any services related to them (except cluster-env and global). */ private void removeOrphanConfigTypes(Configuration configuration) { - Blueprint blueprint = clusterTopology.getBlueprint(); + BlueprintV2 blueprint = clusterTopology.getBlueprint(); Collection<String> configTypes = configuration.getAllConfigTypes(); for (String configType : configTypes) { @@ -162,10 +152,11 @@ public class ClusterConfigurationRequest { Set<String> updatedConfigTypes = new HashSet<>(); Cluster cluster = getCluster(); - Blueprint blueprint = clusterTopology.getBlueprint(); + BlueprintV2 blueprint = clusterTopology.getBlueprint(); - Configuration stackDefaults = blueprint.getStack().getConfiguration(blueprint.getServices()); - Map<String, Map<String, String>> stackDefaultProps = stackDefaults.getProperties(); + //Configuration stackDefaults = blueprint.getStack().getConfiguration(blueprint.getAllServices()); + //Map<String, Map<String, String>> stackDefaultProps = stackDefaults.getProperties(); + Map<String, Map<String, String>> stackDefaultProps = new Configuration(new HashMap<>(), new HashMap<>()).getProperties(); // add clusterHostInfo containing components to hosts map, based on Topology, to use this one instead of // StageUtils.getClusterInfo() @@ -176,7 +167,7 @@ public class ClusterConfigurationRequest { // generate principals & keytabs for headless identities AmbariContext.getController().getKerberosHelper() .ensureHeadlessIdentities(cluster, existingConfigurations, - new HashSet<>(blueprint.getServices())); + new HashSet(blueprint.getAllServices())); // apply Kerberos specific configurations Map<String, Map<String, String>> updatedConfigs = AmbariContext.getController().getKerberosHelper() @@ -231,17 +222,17 @@ public class ClusterConfigurationRequest { * @param blueprint the blueprint * @return a map of service names to component names */ - private Map<String, Set<String>> createServiceComponentMap(Blueprint blueprint) { + private Map<String, Set<String>> createServiceComponentMap(BlueprintV2 blueprint) { Map<String, Set<String>> serviceComponents = new HashMap<>(); - Collection<String> services = blueprint.getServices(); + Collection<Service> services = blueprint.getAllServices(); if(services != null) { - for (String service : services) { - Collection<String> components = blueprint.getComponents(service); - serviceComponents.put(service, + for (Service service : services) { + Collection<ComponentV2> components = blueprint.getComponents(service); + serviceComponents.put(service.getType(), (components == null) ? Collections.emptySet() - : new HashSet<>(blueprint.getComponents(service))); + : new HashSet(blueprint.getComponents(service))); } } @@ -278,16 +269,16 @@ public class ClusterConfigurationRequest { return propertyHasCustomValue; } - private Map<String, String> createComponentHostMap(Blueprint blueprint) { + private Map<String, String> createComponentHostMap(BlueprintV2 blueprint) { Map<String, String> componentHostsMap = new HashMap<>(); - for (String service : blueprint.getServices()) { - Collection<String> components = blueprint.getComponents(service); - for (String component : components) { - Collection<String> componentHost = clusterTopology.getHostAssignmentsForComponent(component); + for (Service service : blueprint.getAllServices()) { + Collection<ComponentV2> components = blueprint.getComponents(service); + for (ComponentV2 component : components) { + Collection<String> componentHost = clusterTopology.getHostAssignmentsForComponent(component.getType()); // retrieve corresponding clusterInfoKey for component using StageUtils - String clusterInfoKey = StageUtils.getComponentToClusterInfoKeyMap().get(component); + String clusterInfoKey = StageUtils.getComponentToClusterInfoKeyMap().get(component.getType()); if (clusterInfoKey == null) { - clusterInfoKey = component.toLowerCase() + "_hosts"; + clusterInfoKey = component.getType().toLowerCase() + "_hosts"; } componentHostsMap.put(clusterInfoKey, StringUtils.join(componentHost, ",")); } @@ -300,7 +291,7 @@ public class ClusterConfigurationRequest { try { Cluster cluster = getCluster(); - Blueprint blueprint = clusterTopology.getBlueprint(); + BlueprintV2 blueprint = clusterTopology.getBlueprint(); Configuration clusterConfiguration = clusterTopology.getConfiguration(); Map<String, Map<String, String>> existingConfigurations = clusterConfiguration.getFullProperties(); @@ -352,16 +343,16 @@ public class ClusterConfigurationRequest { //todo: also handle setting of host group scoped configuration which is updated by config processor List<BlueprintServiceConfigRequest> configurationRequests = new LinkedList<>(); - Blueprint blueprint = clusterTopology.getBlueprint(); + BlueprintV2 blueprint = clusterTopology.getBlueprint(); Configuration clusterConfiguration = clusterTopology.getConfiguration(); - for (String service : blueprint.getServices()) { + for (Service service : blueprint.getAllServices()) { //todo: remove intermediate request type // one bp config request per service - BlueprintServiceConfigRequest blueprintConfigRequest = new BlueprintServiceConfigRequest(service); - - for (String serviceConfigType : stack.getAllConfigurationTypes(service)) { - Set<String> excludedConfigTypes = stack.getExcludedConfigurationTypes(service); + BlueprintServiceConfigRequest blueprintConfigRequest = new BlueprintServiceConfigRequest(service.getType()); + StackV2 serviceStack = service.getStack(); + for (String serviceConfigType : serviceStack.getAllConfigurationTypes(service.getType())) { + Set<String> excludedConfigTypes = serviceStack.getExcludedConfigurationTypes(service.getType()); if (!excludedConfigTypes.contains(serviceConfigType)) { // skip handling of cluster-env here if (! serviceConfigType.equals("cluster-env")) { http://git-wip-us.apache.org/repos/asf/ambari/blob/65d44cd5/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterTopology.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterTopology.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterTopology.java index 69ccb61..f0d6e59 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterTopology.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterTopology.java @@ -18,12 +18,13 @@ package org.apache.ambari.server.topology; -import java.util.Collection; -import java.util.Map; - import org.apache.ambari.server.controller.RequestStatusResponse; +import org.apache.ambari.server.controller.internal.ConfigurationContext; import org.apache.ambari.server.controller.internal.ProvisionAction; +import java.util.Collection; +import java.util.Map; + /** * Represents a full cluster topology including all instance information as well as the associated * blueprint which provides all abstract topology information. @@ -49,7 +50,7 @@ public interface ClusterTopology { * * @return assocaited blueprint */ - Blueprint getBlueprint(); + BlueprintV2 getBlueprint(); /** * Get the cluster scoped configuration for the cluster. @@ -58,8 +59,12 @@ public interface ClusterTopology { * * @return cluster scoped configuration */ + @Deprecated Configuration getConfiguration(); + + Collection<Service> getServiceConfigs(); + /** * Get host group information. * @@ -118,18 +123,18 @@ public interface ClusterTopology { void addHostToTopology(String hostGroupName, String host) throws InvalidTopologyException, NoSuchHostGroupException; /** - * Determine if NameNode HA is enabled. + * Determine if NameNode HA is enabled within ConfigurationContext. * * @return true if NameNode HA is enabled; false otherwise */ - boolean isNameNodeHAEnabled(); + boolean isNameNodeHAEnabled(ConfigurationContext configuration); /** - * Determine if Yarn ResourceManager HA is enabled. + * Determine if Yarn ResourceManager HA is enabled within ConfigurationContext. * * @return true if Yarn ResourceManager HA is enabled; false otherwise */ - boolean isYarnResourceManagerHAEnabled(); + boolean isYarnResourceManagerHAEnabled(ConfigurationContext configuration); /** * Determine if the cluster is kerberos enabled.
