http://git-wip-us.apache.org/repos/asf/ambari/blob/c9f0dd0b/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintValidatorImpl.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintValidatorImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintValidatorImpl.java new file mode 100644 index 0000000..70d1907 --- /dev/null +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintValidatorImpl.java @@ -0,0 +1,318 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ambari.server.topology; + +import org.apache.ambari.server.controller.internal.Stack; +import org.apache.ambari.server.state.AutoDeployInfo; +import org.apache.ambari.server.state.DependencyInfo; + +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; + +/** + * Default blueprint validator. + */ +public class BlueprintValidatorImpl implements BlueprintValidator { + + private final Blueprint blueprint; + private final Stack stack; + + public BlueprintValidatorImpl(Blueprint blueprint) { + this.blueprint = blueprint; + this.stack = blueprint.getStack(); + } + @Override + public void validateTopology() throws InvalidTopologyException { + Collection<HostGroup> hostGroups = blueprint.getHostGroups().values(); + Map<String, Map<String, Collection<DependencyInfo>>> missingDependencies = + new HashMap<String, Map<String, Collection<DependencyInfo>>>(); + + Collection<String> services = blueprint.getServices(); + for (HostGroup group : hostGroups) { + Map<String, Collection<DependencyInfo>> missingGroupDependencies = validateHostGroup(group); + if (! missingGroupDependencies.isEmpty()) { + missingDependencies.put(group.getName(), missingGroupDependencies); + } + } + + Collection<String> cardinalityFailures = new HashSet<String>(); + for (String service : services) { + for (String component : stack.getComponents(service)) { + Cardinality cardinality = stack.getCardinality(component); + AutoDeployInfo autoDeploy = stack.getAutoDeployInfo(component); + if (cardinality.isAll()) { + cardinalityFailures.addAll(verifyComponentInAllHostGroups(component, autoDeploy)); + } else { + cardinalityFailures.addAll(verifyComponentCardinalityCount( + component, cardinality, autoDeploy)); + } + } + } + + if (! missingDependencies.isEmpty() || ! cardinalityFailures.isEmpty()) { + generateInvalidTopologyException(missingDependencies, cardinalityFailures); + } + } + + @Override + public void validateRequiredProperties() throws InvalidTopologyException { + //todo: combine with RequiredPasswordValidator + Map<String, Map<String, Collection<String>>> missingProperties = + new HashMap<String, Map<String, Collection<String>>>(); + + // we don't want to include default stack properties so we can't just use hostGroup full properties + Map<String, Map<String, String>> clusterConfigurations = blueprint.getConfiguration().getProperties(); + + for (HostGroup hostGroup : blueprint.getHostGroups().values()) { + Collection<String> processedServices = new HashSet<String>(); + Map<String, Collection<String>> allRequiredProperties = new HashMap<String, Collection<String>>(); + Map<String, Map<String, String>> operationalConfiguration = new HashMap<String, Map<String, String>>(clusterConfigurations); + + operationalConfiguration.putAll(hostGroup.getConfiguration().getProperties()); + for (String component : hostGroup.getComponents()) { + //check that MYSQL_SERVER component is not available while hive is using existing db + if (component.equals("MYSQL_SERVER")) { + Map<String, String> hiveEnvConfig = clusterConfigurations.get("hive-env"); + if (hiveEnvConfig != null && !hiveEnvConfig.isEmpty() && hiveEnvConfig.get("hive_database") != null + && hiveEnvConfig.get("hive_database").startsWith("Existing")) { + throw new IllegalArgumentException("Incorrect configuration: MYSQL_SERVER component is available but hive" + + " using existing db!"); + } + } + + //for now, AMBARI is not recognized as a service in Stacks + if (! component.equals("AMBARI_SERVER")) { + String serviceName = stack.getServiceForComponent(component); + if (processedServices.add(serviceName)) { + Collection<Stack.ConfigProperty> requiredServiceConfigs = + stack.getRequiredConfigurationProperties(serviceName); + + for (Stack.ConfigProperty requiredConfig : requiredServiceConfigs) { + String configCategory = requiredConfig.getType(); + String propertyName = requiredConfig.getName(); + if (! stack.isPasswordProperty(serviceName, configCategory, propertyName)) { + Collection<String> typeRequirements = allRequiredProperties.get(configCategory); + if (typeRequirements == null) { + typeRequirements = new HashSet<String>(); + allRequiredProperties.put(configCategory, typeRequirements); + } + typeRequirements.add(propertyName); + } + } + } + } + } + for (Map.Entry<String, Collection<String>> requiredTypeProperties : allRequiredProperties.entrySet()) { + String requiredCategory = requiredTypeProperties.getKey(); + Collection<String> requiredProperties = requiredTypeProperties.getValue(); + Collection<String> operationalTypeProps = operationalConfiguration.containsKey(requiredCategory) ? + operationalConfiguration.get(requiredCategory).keySet() : + Collections.<String>emptyList(); + + requiredProperties.removeAll(operationalTypeProps); + if (! requiredProperties.isEmpty()) { + String hostGroupName = hostGroup.getName(); + Map<String, Collection<String>> hostGroupMissingProps = missingProperties.get(hostGroupName); + if (hostGroupMissingProps == null) { + hostGroupMissingProps = new HashMap<String, Collection<String>>(); + missingProperties.put(hostGroupName, hostGroupMissingProps); + } + hostGroupMissingProps.put(requiredCategory, requiredProperties); + } + } + } + + if (! missingProperties.isEmpty()) { + throw new InvalidTopologyException("Missing required properties. Specify a value for these " + + "properties in the blueprint configuration. " + missingProperties); + } + } + + /** + * Verify that a component is included in all host groups. + * For components that are auto-install enabled, will add component to topology if needed. + * + * @param component component to validate + * @param autoDeploy auto-deploy information for component + * + * @return collection of missing component information + */ + private Collection<String> verifyComponentInAllHostGroups(String component, AutoDeployInfo autoDeploy) { + + Collection<String> cardinalityFailures = new HashSet<String>(); + int actualCount = blueprint.getHostGroupsForComponent(component).size(); + Map<String, HostGroup> hostGroups = blueprint.getHostGroups(); + if (actualCount != hostGroups.size()) { + if (autoDeploy != null && autoDeploy.isEnabled()) { + for (HostGroup group : hostGroups.values()) { + group.addComponent(component); + } + } else { + cardinalityFailures.add(component + "(actual=" + actualCount + ", required=ALL)"); + } + } + return cardinalityFailures; + } + + private Map<String, Collection<DependencyInfo>> validateHostGroup(HostGroup group) { + Map<String, Collection<DependencyInfo>> missingDependencies = + new HashMap<String, Collection<DependencyInfo>>(); + + Collection<String> blueprintServices = blueprint.getServices(); + Collection<String> groupComponents = group.getComponents(); + for (String component : new HashSet<String>(groupComponents)) { + Collection<DependencyInfo> dependenciesForComponent = stack.getDependenciesForComponent(component); + for (DependencyInfo dependency : dependenciesForComponent) { + String conditionalService = stack.getConditionalServiceForDependency(dependency); + if (conditionalService != null && ! blueprintServices.contains(conditionalService)) { + continue; + } + + String dependencyScope = dependency.getScope(); + String componentName = dependency.getComponentName(); + AutoDeployInfo autoDeployInfo = dependency.getAutoDeploy(); + boolean resolved = false; + + if (dependencyScope.equals("cluster")) { + Collection<String> missingDependencyInfo = verifyComponentCardinalityCount( + componentName, new Cardinality("1+"), autoDeployInfo); + + resolved = missingDependencyInfo.isEmpty(); + } else if (dependencyScope.equals("host")) { + if (groupComponents.contains(component) || (autoDeployInfo != null && autoDeployInfo.isEnabled())) { + resolved = true; + group.addComponent(componentName); + } + } + + if (! resolved) { + Collection<DependencyInfo> missingCompDependencies = missingDependencies.get(component); + if (missingCompDependencies == null) { + missingCompDependencies = new HashSet<DependencyInfo>(); + missingDependencies.put(component, missingCompDependencies); + } + missingCompDependencies.add(dependency); + } + } + } + return missingDependencies; + } + + /** + * Verify that a component meets cardinality requirements. For components that are + * auto-install enabled, will add component to topology if needed. + * + * @param component component to validate + * @param cardinality required cardinality + * @param autoDeploy auto-deploy information for component + * + * @return collection of missing component information + */ + public Collection<String> verifyComponentCardinalityCount(String component, + Cardinality cardinality, + AutoDeployInfo autoDeploy) { + + Map<String, Map<String, String>> configProperties = blueprint.getConfiguration().getProperties(); + Collection<String> cardinalityFailures = new HashSet<String>(); + //todo: don't hard code this HA logic here + if (ClusterTopologyImpl.isNameNodeHAEnabled(configProperties) && + (component.equals("SECONDARY_NAMENODE"))) { + // override the cardinality for this component in an HA deployment, + // since the SECONDARY_NAMENODE should not be started in this scenario + cardinality = new Cardinality("0"); + } + + int actualCount = blueprint.getHostGroupsForComponent(component).size(); + if (! cardinality.isValidCount(actualCount)) { + boolean validated = ! isDependencyManaged(stack, component, configProperties); + if (! validated && autoDeploy != null && autoDeploy.isEnabled() && cardinality.supportsAutoDeploy()) { + String coLocateName = autoDeploy.getCoLocate(); + if (coLocateName != null && ! coLocateName.isEmpty()) { + Collection<HostGroup> coLocateHostGroups = blueprint.getHostGroupsForComponent(coLocateName.split("/")[1]); + if (! coLocateHostGroups.isEmpty()) { + validated = true; + HostGroup group = coLocateHostGroups.iterator().next(); + group.addComponent(component); + + } + } + } + if (! validated) { + cardinalityFailures.add(component + "(actual=" + actualCount + ", required=" + + cardinality.getValue() + ")"); + } + } + return cardinalityFailures; + } + + /** + * Determine if a component is managed, meaning that it is running inside of the cluster + * topology. Generally, non-managed dependencies will be database components. + * + * @param stack stack instance + * @param component component to determine if it is managed + * @param clusterConfig cluster configuration + * + * @return true if the specified component managed by the cluster; false otherwise + */ + protected boolean isDependencyManaged(Stack stack, String component, Map<String, Map<String, String>> clusterConfig) { + boolean isManaged = true; + String externalComponentConfig = stack.getExternalComponentConfig(component); + if (externalComponentConfig != null) { + String[] toks = externalComponentConfig.split("/"); + String externalComponentConfigType = toks[0]; + String externalComponentConfigProp = toks[1]; + Map<String, String> properties = clusterConfig.get(externalComponentConfigType); + if (properties != null && properties.containsKey(externalComponentConfigProp)) { + if (properties.get(externalComponentConfigProp).startsWith("Existing")) { + isManaged = false; + } + } + } + return isManaged; + } + + /** + * Generate an exception for topology validation failure. + * + * @param missingDependencies missing dependency information + * @param cardinalityFailures missing service component information + * + * @throws IllegalArgumentException Always thrown and contains information regarding the topology validation failure + * in the msg + */ + private void generateInvalidTopologyException(Map<String, Map<String, Collection<DependencyInfo>>> missingDependencies, + Collection<String> cardinalityFailures) throws InvalidTopologyException { + + //todo: encapsulate some of this in exception? + String msg = "Cluster Topology validation failed."; + if (! cardinalityFailures.isEmpty()) { + msg += " Invalid service component count: " + cardinalityFailures; + } + if (! missingDependencies.isEmpty()) { + msg += " Unresolved component dependencies: " + missingDependencies; + } + msg += ". To disable topology validation and create the blueprint, " + + "add the following to the end of the url: '?validate_topology=false'"; + throw new InvalidTopologyException(msg); + } +}
http://git-wip-us.apache.org/repos/asf/ambari/blob/c9f0dd0b/ambari-server/src/main/java/org/apache/ambari/server/topology/Cardinality.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/Cardinality.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/Cardinality.java new file mode 100644 index 0000000..666b1bd --- /dev/null +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/Cardinality.java @@ -0,0 +1,90 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ambari.server.topology; + +/** + * Component cardinality representation. + */ +public class Cardinality { + String cardinality; + int min = 0; + int max = Integer.MAX_VALUE; + int exact = -1; + boolean isAll = false; + + public Cardinality(String cardinality) { + this.cardinality = cardinality; + if (cardinality != null && ! cardinality.isEmpty()) { + if (cardinality.contains("+")) { + min = Integer.valueOf(cardinality.split("\\+")[0]); + } else if (cardinality.contains("-")) { + String[] toks = cardinality.split("-"); + min = Integer.parseInt(toks[0]); + max = Integer.parseInt(toks[1]); + } else if (cardinality.equals("ALL")) { + isAll = true; + } else { + exact = Integer.parseInt(cardinality); + } + } + } + + /** + * Determine if component is required for all host groups. + * + * @return true if cardinality is 'ALL', false otherwise + */ + public boolean isAll() { + return isAll; + } + + /** + * Determine if the given count satisfies the required cardinality. + * + * @param count number of host groups containing component + * + * @return true id count satisfies the required cardinality, false otherwise + */ + public boolean isValidCount(int count) { + if (isAll) { + return false; + } else if (exact != -1) { + return count == exact; + } else return count >= min && count <= max; + } + + /** + * Determine if the cardinality count supports auto-deployment. + * This determination is independent of whether the component is configured + * to be auto-deployed. This only indicates whether auto-deployment is + * supported for the current cardinality. + * + * At this time, only cardinalities of ALL or where a count of 1 is valid are + * supported. + * + * @return true if cardinality supports auto-deployment + */ + public boolean supportsAutoDeploy() { + return isValidCount(1) || isAll; + } + + public String getValue() { + return cardinality; + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/c9f0dd0b/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterConfigurationRequest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterConfigurationRequest.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterConfigurationRequest.java new file mode 100644 index 0000000..1bffbf2 --- /dev/null +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterConfigurationRequest.java @@ -0,0 +1,271 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ambari.server.topology; + +import org.apache.ambari.server.AmbariException; +import org.apache.ambari.server.controller.AmbariManagementController; +import org.apache.ambari.server.controller.AmbariServer; +import org.apache.ambari.server.controller.ClusterRequest; +import org.apache.ambari.server.controller.ConfigurationRequest; +import org.apache.ambari.server.controller.internal.AbstractResourceProvider; +import org.apache.ambari.server.controller.internal.BlueprintConfigurationProcessor; +import org.apache.ambari.server.controller.internal.ClusterResourceProvider; +import org.apache.ambari.server.controller.internal.ConfigurationTopologyException; +import org.apache.ambari.server.controller.internal.Stack; +import org.apache.ambari.server.state.SecurityType; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * Responsible for cluster configuration. + */ +public class ClusterConfigurationRequest { + + protected final static Logger LOG = LoggerFactory.getLogger(ClusterConfigurationRequest.class); + + private ClusterTopology clusterTopology; + private BlueprintConfigurationProcessor configurationProcessor; + private AmbariManagementController controller = AmbariServer.getController(); + private Stack stack; + + public ClusterConfigurationRequest(ClusterTopology clusterTopology) throws AmbariException { + Blueprint blueprint = clusterTopology.getBlueprint(); + this.stack = blueprint.getStack(); + this.clusterTopology = clusterTopology; + // set initial configuration (not topology resolved) + this.configurationProcessor = new BlueprintConfigurationProcessor(clusterTopology); + setConfigurationsOnCluster(clusterTopology, "INITIAL"); + } + + // get names of required host groups + public Collection<String> getRequiredHostGroups() { + return configurationProcessor.getRequiredHostGroups(); + } + + public void process() throws AmbariException, ConfigurationTopologyException { + // this will update the topo cluster config and all host group configs in the cluster topology + configurationProcessor.doUpdateForClusterCreate(); + setConfigurationsOnCluster(clusterTopology, "TOPOLOGY_RESOLVED"); + } + + /** + * Set all configurations on the cluster resource. + * @param clusterTopology cluster topology + * @param tag config tag + * + * @throws AmbariException unable to set config on cluster + */ + public void setConfigurationsOnCluster(ClusterTopology clusterTopology, String tag) throws AmbariException { + //todo: also handle setting of host group scoped configuration which is updated by config processor + List<BlueprintServiceConfigRequest> listofConfigRequests = new LinkedList<BlueprintServiceConfigRequest>(); + + Blueprint blueprint = clusterTopology.getBlueprint(); + Configuration clusterConfiguration = clusterTopology.getConfiguration(); + + for (String service : blueprint.getServices()) { + //todo: remove intermediate request type + // one bp config request per service + BlueprintServiceConfigRequest blueprintConfigRequest = new BlueprintServiceConfigRequest(service); + + for (String serviceConfigType : stack.getAllConfigurationTypes(service)) { + Set<String> excludedConfigTypes = stack.getExcludedConfigurationTypes(service); + if (!excludedConfigTypes.contains(serviceConfigType)) { + // skip handling of cluster-env here + if (! serviceConfigType.equals("cluster-env")) { + if (clusterConfiguration.getFullProperties().containsKey(serviceConfigType)) { + blueprintConfigRequest.addConfigElement(serviceConfigType, + clusterConfiguration.getFullProperties().get(serviceConfigType), + clusterConfiguration.getFullAttributes().get(serviceConfigType)); + } + } + } + } + + listofConfigRequests.add(blueprintConfigRequest); + } + + // since the stack returns "cluster-env" with each service's config ensure that only one + // ClusterRequest occurs for the global cluster-env configuration + BlueprintServiceConfigRequest globalConfigRequest = new BlueprintServiceConfigRequest("GLOBAL-CONFIG"); + Map<String, String> clusterEnvProps = clusterConfiguration.getFullProperties().get("cluster-env"); + Map<String, Map<String, String>> clusterEnvAttributes = clusterConfiguration.getFullAttributes().get("cluster-env"); + + globalConfigRequest.addConfigElement("cluster-env", clusterEnvProps,clusterEnvAttributes); + listofConfigRequests.add(globalConfigRequest); + + setConfigurationsOnCluster(listofConfigRequests, tag); + } + + /** + * Creates a ClusterRequest for each service that + * includes any associated config types and configuration. The Blueprints + * implementation will now create one ClusterRequest per service, in order + * to comply with the ServiceConfigVersioning framework in Ambari. + * + * This method will also send these requests to the management controller. + * + * @param listOfBlueprintConfigRequests a list of requests to send to the AmbariManagementController. + * + * @throws AmbariException upon any error that occurs during updateClusters + */ + private void setConfigurationsOnCluster(List<BlueprintServiceConfigRequest> listOfBlueprintConfigRequests, + String tag) throws AmbariException { + // iterate over services to deploy + for (BlueprintServiceConfigRequest blueprintConfigRequest : listOfBlueprintConfigRequests) { + ClusterRequest clusterRequest = null; + // iterate over the config types associated with this service + List<ConfigurationRequest> requestsPerService = new LinkedList<ConfigurationRequest>(); + for (BlueprintServiceConfigElement blueprintElement : blueprintConfigRequest.getConfigElements()) { + Map<String, Object> clusterProperties = new HashMap<String, Object>(); + clusterProperties.put(ClusterResourceProvider.CLUSTER_NAME_PROPERTY_ID, clusterTopology.getClusterName()); + clusterProperties.put(ClusterResourceProvider.CLUSTER_DESIRED_CONFIGS_PROPERTY_ID + "/type", blueprintElement.getTypeName()); + clusterProperties.put(ClusterResourceProvider.CLUSTER_DESIRED_CONFIGS_PROPERTY_ID + "/tag", tag); + for (Map.Entry<String, String> entry : blueprintElement.getConfiguration().entrySet()) { + clusterProperties.put(ClusterResourceProvider.CLUSTER_DESIRED_CONFIGS_PROPERTY_ID + + "/properties/" + entry.getKey(), entry.getValue()); + } + if (blueprintElement.getAttributes() != null) { + for (Map.Entry<String, Map<String, String>> attribute : blueprintElement.getAttributes().entrySet()) { + String attributeName = attribute.getKey(); + for (Map.Entry<String, String> attributeOccurrence : attribute.getValue().entrySet()) { + clusterProperties.put(ClusterResourceProvider.CLUSTER_DESIRED_CONFIGS_PROPERTY_ID + "/properties_attributes/" + + attributeName + "/" + attributeOccurrence.getKey(), attributeOccurrence.getValue()); + } + } + } + + // only create one cluster request per service, which includes + // all the configuration types for that service + if (clusterRequest == null) { + SecurityType securityType; + String requestedSecurityType = (String) clusterProperties.get( + ClusterResourceProvider.CLUSTER_SECURITY_TYPE_PROPERTY_ID); + if(requestedSecurityType == null) + securityType = null; + else { + try { + securityType = SecurityType.valueOf(requestedSecurityType.toUpperCase()); + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException(String.format( + "Cannot set cluster security type to invalid value: %s", requestedSecurityType)); + } + } + + clusterRequest = new ClusterRequest( + (Long) clusterProperties.get(ClusterResourceProvider.CLUSTER_ID_PROPERTY_ID), + (String) clusterProperties.get(ClusterResourceProvider.CLUSTER_NAME_PROPERTY_ID), + (String) clusterProperties.get(ClusterResourceProvider.CLUSTER_PROVISIONING_STATE_PROPERTY_ID), + securityType, + (String) clusterProperties.get(ClusterResourceProvider.CLUSTER_VERSION_PROPERTY_ID), + null); + } + + //todo: made getConfigurationRequests static so that I could access from here, where does it belong? + List<ConfigurationRequest> listOfRequests = + AbstractResourceProvider.getConfigurationRequests("Clusters", clusterProperties); + requestsPerService.addAll(listOfRequests); + } + + // set total list of config requests, including all config types for this service + if (clusterRequest != null) { + clusterRequest.setDesiredConfig(requestsPerService); + LOG.info("Sending cluster config update request for service = " + blueprintConfigRequest.getServiceName()); + controller.updateClusters(Collections.singleton(clusterRequest), null); + } else { + LOG.error("ClusterRequest should not be null for service = " + blueprintConfigRequest.getServiceName()); + } + } + } + + /** + * Internal class meant to represent the collection of configuration + * items and configuration attributes that are associated with a given service. + * + * This class is used to support proper configuration versioning when + * Ambari Blueprints is used to deploy a cluster. + */ + private static class BlueprintServiceConfigRequest { + + private final String serviceName; + + private List<BlueprintServiceConfigElement> configElements = + new LinkedList<BlueprintServiceConfigElement>(); + + BlueprintServiceConfigRequest(String serviceName) { + this.serviceName = serviceName; + } + + void addConfigElement(String type, Map<String, String> props, Map<String, Map<String, String>> attributes) { + if (props == null) { + props = Collections.emptyMap(); + } + + if (attributes == null) { + attributes = Collections.emptyMap(); + } + configElements.add(new BlueprintServiceConfigElement(type, props, attributes)); + } + + public String getServiceName() { + return serviceName; + } + + List<BlueprintServiceConfigElement> getConfigElements() { + return configElements; + } + } + + /** + * Internal class that represents the configuration + * and attributes for a given configuration type. + */ + private static class BlueprintServiceConfigElement { + private final String typeName; + + private final Map<String, String> configuration; + + private final Map<String, Map<String, String>> attributes; + + BlueprintServiceConfigElement(String type, Map<String, String> props, Map<String, Map<String, String>> attributes) { + this.typeName = type; + this.configuration = props; + this.attributes = attributes; + } + + public String getTypeName() { + return typeName; + } + + public Map<String, String> getConfiguration() { + return configuration; + } + + public Map<String, Map<String, String>> getAttributes() { + return attributes; + } + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/c9f0dd0b/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterTopology.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterTopology.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterTopology.java new file mode 100644 index 0000000..e924653 --- /dev/null +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterTopology.java @@ -0,0 +1,116 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ambari.server.topology; + +import java.util.Collection; +import java.util.Map; + +/** + * Represents a full cluster topology including all instance information as well as the associated + * blueprint which provides all abstract topology information. + */ +public interface ClusterTopology { + + /** + * Get the name of the cluster. + * + * @return cluster name + */ + public String getClusterName(); + + /** + * Get the blueprint associated with the cluster. + * + * @return assocaited blueprint + */ + public Blueprint getBlueprint(); + + /** + * Get the cluster scoped configuration for the cluster. + * This configuration has the blueprint cluster scoped + * configuration set as it's parent. + * + * @return cluster scoped configuration + */ + public Configuration getConfiguration(); + + /** + * Get host group information. + * + * @return map of host group name to host group information + */ + public Map<String, HostGroupInfo> getHostGroupInfo(); + + /** + * Get the names of all of host groups which contain the specified component. + * + * @param component component name + * + * @return collection of host group names which contain the specified component + */ + public Collection<String> getHostGroupsForComponent(String component); + + /** + * Get the name of the host group which is mapped to the specified host. + * + * @param hostname host name + * + * @return name of the host group which is mapped to the specified host or null if + * no group is mapped to the host + */ + public String getHostGroupForHost(String hostname); + + /** + * Get all hosts which are mapped to a host group which contains the specified component. + * The host need only to be mapped to the hostgroup, not actually provisioned. + * + * @param component component name + * + * @return collection of hosts for the specified component; will not return null + */ + public Collection<String> getHostAssignmentsForComponent(String component); + + /** + * Update the existing topology based on the provided topology request. + * + * @param topologyRequest request modifying the topology + * + * @throws InvalidTopologyException if the request specified invalid topology information or if + * making the requested changes would result in an invalid topology + */ + public void update(TopologyRequest topologyRequest) throws InvalidTopologyException; + + /** + * Add a new host to the topology. + * + * @param hostGroupName name of associated host group + * @param host name of host + * + * @throws InvalidTopologyException if the host being added is already registered to a different host group + * @throws NoSuchHostGroupException if the specified host group is invalid + */ + public void addHostToTopology(String hostGroupName, String host) throws InvalidTopologyException, NoSuchHostGroupException; + + /** + * Determine if NameNode HA is enabled. + * + * @return true if NameNode HA is enabled; false otherwise + */ + public boolean isNameNodeHAEnabled(); +} http://git-wip-us.apache.org/repos/asf/ambari/blob/c9f0dd0b/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterTopologyImpl.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterTopologyImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterTopologyImpl.java new file mode 100644 index 0000000..84e90bf --- /dev/null +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterTopologyImpl.java @@ -0,0 +1,245 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distribut + * ed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ambari.server.topology; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * Represents a cluster topology. + * Topology includes the the associated blueprint, cluster configuration and hostgroup -> host mapping. + */ +public class ClusterTopologyImpl implements ClusterTopology { + + private String clusterName; + //todo: currently topology is only associated with a single bp + //todo: this will need to change to allow usage of multiple bp's for the same cluster + //todo: for example: provision using bp1 and scale using bp2 + private Blueprint blueprint; + private Configuration configuration; + private final Map<String, HostGroupInfo> hostGroupInfoMap = + new HashMap<String, HostGroupInfo>(); + + + //todo: will need to convert all usages of hostgroup name to use fully qualified name (BP/HG) + //todo: for now, restrict scaling to the same BP + public ClusterTopologyImpl(TopologyRequest topologyRequest) throws InvalidTopologyException { + this.clusterName = topologyRequest.getClusterName(); + // provision cluster currently requires that all hostgroups have same BP so it is ok to use root level BP here + this.blueprint = topologyRequest.getBlueprint(); + this.configuration = topologyRequest.getConfiguration(); + + registerHostGroupInfo(topologyRequest.getHostGroupInfo()); + + validateTopology(topologyRequest.getTopologyValidators()); + } + + //todo: only used in tests, remove. Validators not invoked when this constructor is used. + public ClusterTopologyImpl(String clusterName, + Blueprint blueprint, + Configuration configuration, + Map<String, HostGroupInfo> hostGroupInfo) + throws InvalidTopologyException { + + this.clusterName = clusterName; + this.blueprint = blueprint; + this.configuration = configuration; + + registerHostGroupInfo(hostGroupInfo); + } + + @Override + public void update(TopologyRequest topologyRequest) throws InvalidTopologyException { + registerHostGroupInfo(topologyRequest.getHostGroupInfo()); + } + + @Override + public String getClusterName() { + return clusterName; + } + + @Override + public Blueprint getBlueprint() { + return blueprint; + } + + @Override + public Configuration getConfiguration() { + return configuration; + } + + @Override + public Map<String, HostGroupInfo> getHostGroupInfo() { + return hostGroupInfoMap; + } + + //todo: do we want to return groups with no requested hosts? + @Override + public Collection<String> getHostGroupsForComponent(String component) { + Collection<String> resultGroups = new ArrayList<String>(); + for (HostGroup group : getBlueprint().getHostGroups().values() ) { + if (group.getComponents().contains(component)) { + resultGroups.add(group.getName()); + } + } + return resultGroups; + } + + @Override + public String getHostGroupForHost(String hostname) { + for (HostGroupInfo groupInfo : hostGroupInfoMap.values() ) { + if (groupInfo.getHostNames().contains(hostname)) { + // a host can only be associated with a single host group + return groupInfo.getHostGroupName(); + } + } + return null; + } + + //todo: host info? + @Override + public void addHostToTopology(String hostGroupName, String host) throws InvalidTopologyException, NoSuchHostGroupException { + if (blueprint.getHostGroup(hostGroupName) == null) { + throw new NoSuchHostGroupException("Attempted to add host to non-existing host group: " + hostGroupName); + } + + // check for host duplicates + String groupContainsHost = getHostGroupForHost(host); + // in case of reserved host, hostgroup will already contain host + if (groupContainsHost != null && ! hostGroupName.equals(groupContainsHost)) { + throw new InvalidTopologyException(String.format( + "Attempted to add host '%s' to hostgroup '%s' but it is already associated with hostgroup '%s'.", + host, hostGroupName, groupContainsHost)); + } + + synchronized(hostGroupInfoMap) { + HostGroupInfo existingHostGroupInfo = hostGroupInfoMap.get(hostGroupName); + if (existingHostGroupInfo == null) { + throw new RuntimeException(String.format("An attempt was made to add host '%s' to an unregistered hostgroup '%s'", + host, hostGroupName)); + } + // ok to add same host multiple times to same group + existingHostGroupInfo.addHost(host); + } + } + + @Override + public Collection<String> getHostAssignmentsForComponent(String component) { + //todo: ordering requirements? + Collection<String> hosts = new ArrayList<String>(); + Collection<String> hostGroups = getHostGroupsForComponent(component); + for (String group : hostGroups) { + hosts.addAll(getHostGroupInfo().get(group).getHostNames()); + } + return hosts; + } + + @Override + public boolean isNameNodeHAEnabled() { + return isNameNodeHAEnabled(configuration.getFullProperties()); + } + + public static boolean isNameNodeHAEnabled(Map<String, Map<String, String>> configurationProperties) { + return configurationProperties.containsKey("hdfs-site") && + configurationProperties.get("hdfs-site").containsKey("dfs.nameservices"); + } + + private void validateTopology(List<TopologyValidator> validators) + throws InvalidTopologyException { + + for (TopologyValidator validator : validators) { + validator.validate(this); + } + } + + private void registerHostGroupInfo(Map<String, HostGroupInfo> groupInfoMap) throws InvalidTopologyException { + checkForDuplicateHosts(groupInfoMap); + for (HostGroupInfo hostGroupInfo : groupInfoMap.values() ) { + String hostGroupName = hostGroupInfo.getHostGroupName(); + //todo: doesn't support using a different blueprint for update (scaling) + HostGroup baseHostGroup = getBlueprint().getHostGroup(hostGroupName); + if (baseHostGroup == null) { + throw new IllegalArgumentException("Invalid host_group specified: " + hostGroupName + + ". All request host groups must have a corresponding host group in the specified blueprint"); + } + //todo: split into two methods + HostGroupInfo existingHostGroupInfo = hostGroupInfoMap.get(hostGroupName); + if (existingHostGroupInfo == null) { + // blueprint host group config + Configuration bpHostGroupConfig = baseHostGroup.getConfiguration(); + // parent config is BP host group config but with parent set to topology cluster scoped config + Configuration parentConfiguration = new Configuration(bpHostGroupConfig.getProperties(), + bpHostGroupConfig.getAttributes(), getConfiguration()); + + hostGroupInfo.getConfiguration().setParentConfiguration(parentConfiguration); + hostGroupInfoMap.put(hostGroupName, hostGroupInfo); + } else { + // Update. Either add hosts or increment request count + if (! hostGroupInfo.getHostNames().isEmpty()) { + try { + // this validates that hosts aren't already registered with groups + addHostsToTopology(hostGroupInfo); + } catch (NoSuchHostGroupException e) { + //todo + throw new InvalidTopologyException("Attempted to add hosts to unknown host group: " + hostGroupName); + } + } else { + existingHostGroupInfo.setRequestedCount( + existingHostGroupInfo.getRequestedHostCount() + hostGroupInfo.getRequestedHostCount()); + } + //todo: throw exception in case where request attempts to modify HG configuration in scaling operation + } + } + } + + private void addHostsToTopology(HostGroupInfo hostGroupInfo) throws InvalidTopologyException, NoSuchHostGroupException { + for (String host: hostGroupInfo.getHostNames()) { + addHostToTopology(hostGroupInfo.getHostGroupName(), host); + } + } + + private void checkForDuplicateHosts(Map<String, HostGroupInfo> groupInfoMap) throws InvalidTopologyException { + Set<String> hosts = new HashSet<String>(); + Set<String> duplicates = new HashSet<String>(); + for (HostGroupInfo group : groupInfoMap.values()) { + // check for duplicates within the new groups + Collection<String> groupHosts = group.getHostNames(); + Collection<String> groupHostsCopy = new HashSet<String>(group.getHostNames()); + groupHostsCopy.retainAll(hosts); + duplicates.addAll(groupHostsCopy); + hosts.addAll(groupHosts); + + // check against existing groups + for (String host : groupHosts) { + if (getHostGroupForHost(host) != null) { + duplicates.add(host); + } + } + } + if (! duplicates.isEmpty()) { + throw new InvalidTopologyException("The following hosts are mapped to multiple host groups: " + duplicates); + } + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/c9f0dd0b/ambari-server/src/main/java/org/apache/ambari/server/topology/Configuration.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/Configuration.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/Configuration.java new file mode 100644 index 0000000..2447b8b --- /dev/null +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/Configuration.java @@ -0,0 +1,187 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distribut + * ed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ambari.server.topology; + +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; + +/** + * Configuration for a topology entity such as a blueprint, hostgroup or cluster. + */ +public class Configuration { + + private Map<String, Map<String, String>> properties; + private Map<String, Map<String, Map<String, String>>> attributes; + + private Configuration parentConfiguration; + + public Configuration(Map<String, Map<String, String>> properties, + Map<String, Map<String, Map<String, String>>> attributes, + Configuration parentConfiguration) { + + this.properties = properties; + this.attributes = attributes; + this.parentConfiguration = parentConfiguration; + + //todo: warning for deprecated global properties + // String message = null; +// for (BlueprintConfigEntity blueprintConfig: blueprint.getConfigurations()){ +// if(blueprintConfig.getType().equals("global")){ +// message = "WARNING: Global configurations are deprecated, please use *-env"; +// break; +// } +// } + } + + public Configuration(Map<String, Map<String, String>> properties, + Map<String, Map<String, Map<String, String>>> attributes) { + + this.properties = properties; + this.attributes = attributes; + } + + public Map<String, Map<String, String>> getProperties() { + return properties; + } + + public Map<String, Map<String, String>> getFullProperties() { + return getFullProperties(Integer.MAX_VALUE); + } + + //re-calculated each time in case parent properties changed + public Map<String, Map<String, String>> getFullProperties(int depthLimit) { + + if (depthLimit == 0) { + return new HashMap<String, Map<String, String>>(properties); + } + + Map<String, Map<String, String>> mergedProperties = parentConfiguration == null ? + new HashMap<String, Map<String, String>>() : + new HashMap<String, Map<String, String>>(parentConfiguration.getFullProperties(--depthLimit)); + + for (Map.Entry<String, Map<String, String>> entry : properties.entrySet()) { + String configType = entry.getKey(); + Map<String, String> typeProps = entry.getValue(); + + if (mergedProperties.containsKey(configType)) { + mergedProperties.get(configType).putAll(typeProps); + } else { + mergedProperties.put(configType, typeProps); + } + } + return mergedProperties; + } + + public Map<String, Map<String, Map<String, String>>> getAttributes() { + return attributes; + } + + //re-calculate each time in case parent properties changed + // attribute structure is very confusing. {type -> {attributeName -> {propName, attributeValue}}} + public Map<String, Map<String, Map<String, String>>> getFullAttributes() { + Map<String, Map<String, Map<String, String>>> mergedAttributeMap = parentConfiguration == null ? + new HashMap<String, Map<String, Map<String, String>>>() : + new HashMap<String, Map<String, Map<String, String>>>(parentConfiguration.getFullAttributes()); + + for (Map.Entry<String, Map<String, Map<String, String>>> typeEntry : attributes.entrySet()) { + String type = typeEntry.getKey(); + if (! mergedAttributeMap.containsKey(type)) { + mergedAttributeMap.put(type, typeEntry.getValue()); + } else { + Map<String, Map<String, String>> mergedAttributes = mergedAttributeMap.get(type); + for (Map.Entry<String, Map<String, String>> attributeEntry : typeEntry.getValue().entrySet()) { + String attribute = attributeEntry.getKey(); + if (! mergedAttributes.containsKey(attribute)) { + mergedAttributes.put(attribute, attributeEntry.getValue()); + } else { + Map<String, String> mergedAttributeProps = mergedAttributes.get(attribute); + for (Map.Entry<String, String> propEntry : attributeEntry.getValue().entrySet()) { + mergedAttributeProps.put(propEntry.getKey(), propEntry.getValue()); + } + } + } + } + } + + mergedAttributeMap.putAll(attributes); + + return mergedAttributeMap; + } + + public Collection<String> getAllConfigTypes() { + Collection<String> allTypes = new HashSet<String>(); + for (String type : getFullProperties().keySet()) { + allTypes.add(type); + } + + for (String type : getFullAttributes().keySet()) { + allTypes.add(type); + } + + return allTypes; + } + + public Configuration getParentConfiguration() { + return parentConfiguration; + } + + public void setParentConfiguration(Configuration parent) { + parentConfiguration = parent; + } + + public String getPropertyValue(String configType, String propertyName) { + return properties.containsKey(configType) ? + properties.get(configType).get(propertyName) : null; + } + + public boolean containsProperty(String configType, String propertyName) { + return properties.containsKey(configType) && properties.get(configType).containsKey(propertyName); + } + + public String setProperty(String configType, String propertyName, String value) { + Map<String, String> typeProperties = properties.get(configType); + if (typeProperties == null) { + typeProperties = new HashMap<String, String>(); + properties.put(configType, typeProperties); + } + + return typeProperties.put(propertyName, value); + } + + // attribute structure is very confusing: {type -> {attributeName -> {propName, attributeValue}}} + public String setAttribute(String configType, String propertyName, String attributeName, String attributeValue) { + Map<String, Map<String, String>> typeAttributes = attributes.get(configType); + if (typeAttributes == null) { + typeAttributes = new HashMap<String, Map<String, String>>(); + attributes.put(configType, typeAttributes); + } + + Map<String, String> attributes = typeAttributes.get(attributeName); + if (attributes == null) { + attributes = new HashMap<String, String>(); + typeAttributes.put(attributeName, attributes); + } + + return attributes.put(propertyName, attributeValue); + } + +} http://git-wip-us.apache.org/repos/asf/ambari/blob/c9f0dd0b/ambari-server/src/main/java/org/apache/ambari/server/topology/ConfigurationFactory.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/ConfigurationFactory.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/ConfigurationFactory.java new file mode 100644 index 0000000..f4dc879 --- /dev/null +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/ConfigurationFactory.java @@ -0,0 +1,121 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distribut + * ed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ambari.server.topology; + +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; + +/** + * Creates a configuration instance from user provided properties. + * Supports both forms of configuration syntax. + * todo: document both forms here +*/ +public class ConfigurationFactory { + + private static final String SCHEMA_IS_NOT_SUPPORTED_MESSAGE = + "Provided configuration format is not supported"; + + public Configuration getConfiguration(Collection<Map<String, String>> configProperties) { + Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>(); + Map<String, Map<String, Map<String, String>>> attributes = new HashMap<String, Map<String, Map<String, String>>>(); + Configuration configuration = new Configuration(properties, attributes); + + if (configProperties != null) { + for (Map<String, String> typeMap : configProperties) { + //todo: can we have a different strategy for each type? + ConfigurationStrategy strategy = decidePopulationStrategy(typeMap); + for (Map.Entry<String, String> entry : typeMap.entrySet()) { + String[] propertyNameTokens = entry.getKey().split("/"); + strategy.setConfiguration(configuration, propertyNameTokens, entry.getValue()); + } + } + } + return configuration; + } + + private ConfigurationStrategy decidePopulationStrategy(Map<String, String> configuration) { + if (configuration != null && !configuration.isEmpty()) { + String keyEntry = configuration.keySet().iterator().next(); + String[] keyNameTokens = keyEntry.split("/"); + int levels = keyNameTokens.length; + String propertiesType = keyNameTokens[1]; + if (levels == 2) { + return new ConfigurationStrategyV1(); + } else if ((levels == 3 && BlueprintFactory.PROPERTIES_PROPERTY_ID.equals(propertiesType)) + || (levels == 4 && BlueprintFactory.PROPERTIES_ATTRIBUTES_PROPERTY_ID.equals(propertiesType))) { + return new ConfigurationStrategyV2(); + } else { + throw new IllegalArgumentException(SCHEMA_IS_NOT_SUPPORTED_MESSAGE); + } + } else { + return new ConfigurationStrategyV2(); + } + } + + /** + * The structure of blueprints is evolving where multiple resource + * structures are to be supported. This class abstracts the population + * of configurations which have changed from a map of key-value strings, + * to an map containing 'properties' and 'properties_attributes' maps. + * + * Extending classes can determine how they want to populate the + * configuration maps depending on input. + */ + private static abstract class ConfigurationStrategy { + + protected abstract void setConfiguration(Configuration configuration, + String[] propertyNameTokens, + String propertyValue); + + } + + /** + * Original blueprint configuration format where configs were a map + * of strings. + */ + protected static class ConfigurationStrategyV1 extends ConfigurationStrategy { + + + @Override + protected void setConfiguration(Configuration configuration, String[] propertyNameTokens, String propertyValue) { + configuration.setProperty(propertyNameTokens[0], propertyNameTokens[1], propertyValue); + } + } + + /** + * New blueprint configuration format where configs are a map from 'properties' and + * 'properties_attributes' to a map of strings. + * + * @since 1.7.0 + */ + protected static class ConfigurationStrategyV2 extends ConfigurationStrategy { + + @Override + protected void setConfiguration(Configuration configuration, String[] propertyNameTokens, String propertyValue) { + String type = propertyNameTokens[0]; + if (BlueprintFactory.PROPERTIES_PROPERTY_ID.equals(propertyNameTokens[1])) { + configuration.setProperty(type, propertyNameTokens[2], propertyValue); + } else if (BlueprintFactory.PROPERTIES_ATTRIBUTES_PROPERTY_ID.equals(propertyNameTokens[1])) { + configuration.setAttribute(type, propertyNameTokens[2], propertyNameTokens[3], propertyValue); + } + } + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/c9f0dd0b/ambari-server/src/main/java/org/apache/ambari/server/topology/HostGroup.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/HostGroup.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/HostGroup.java new file mode 100644 index 0000000..07e3e88 --- /dev/null +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/HostGroup.java @@ -0,0 +1,119 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ambari.server.topology; + +import org.apache.ambari.server.controller.internal.Stack; +import org.apache.ambari.server.orm.entities.HostGroupEntity; +import org.apache.ambari.server.state.DependencyInfo; + +import java.util.Collection; +import java.util.Map; + +/** + * Host Group representation. + */ +public interface HostGroup { + + /** + * Get the name of the host group. + * + * @return the host group name + */ + public String getName(); + + /** + * Get the name of the associated blueprint + * + * @return associated blueprint name + */ + public String getBlueprintName(); + + /** + * Get the fully qualified host group name in the form of + * blueprintName:hostgroupName + * + * @return fully qualified host group name + */ + public String getFullyQualifiedName(); + + /** + * Get all of the host group components. + * + * @return collection of component names + */ + public Collection<String> getComponents(); + + /** + * Get the host group components which belong to the specified service. + * + * @param service service name + * + * @return collection of component names for the specified service; will not return null + */ + public Collection<String> getComponents(String service); + + /** + * Add a component to the host group. + * + * @param component name of the component to add + * + * @return true if the component didn't already exist + */ + public boolean addComponent(String component); + + /** + * Determine if the host group contains a master component. + * + * @return true if the host group contains a master component; false otherwise + */ + public boolean containsMasterComponent(); + + /** + * Get all of the services associated with the host group components. + * + * @return collection of service names + */ + public Collection<String> getServices(); + + /** + * Get the configuration associated with the host group. + * The host group configuration has the blueprint cluster scoped + * configuration set as it's parent. + * + * @return host group configuration + */ + public Configuration getConfiguration(); + + /** + * Get the stack associated with the host group. + * + * @return associated stack + */ + public Stack getStack(); + + /** + * Get the cardinality value that was specified for the host group. + * This is simply meta-data for the stack that a deployer can use + * and this information is not used by ambari. + * + * @return the cardinality specified for the hostgroup + */ + public String getCardinality(); +} + http://git-wip-us.apache.org/repos/asf/ambari/blob/c9f0dd0b/ambari-server/src/main/java/org/apache/ambari/server/topology/HostGroupImpl.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/HostGroupImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/HostGroupImpl.java new file mode 100644 index 0000000..b89e7e4 --- /dev/null +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/HostGroupImpl.java @@ -0,0 +1,239 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distribut + * ed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ambari.server.topology; + +import com.google.gson.Gson; +import org.apache.ambari.server.controller.internal.Stack; +import org.apache.ambari.server.orm.entities.HostGroupComponentEntity; +import org.apache.ambari.server.orm.entities.HostGroupConfigEntity; +import org.apache.ambari.server.orm.entities.HostGroupEntity; + +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +/** + * Host Group implementation. + */ +public class HostGroupImpl implements HostGroup { + + /** + * host group name + */ + private String name; + + /** + * blueprint name + */ + private String blueprintName; + + /** + * components contained in the host group + */ + private Collection<String> components = new HashSet<String>(); + + /** + * map of service to components for the host group + */ + private Map<String, Set<String>> componentsForService = new HashMap<String, Set<String>>(); + + /** + * configuration + */ + private Configuration configuration = null; + + private boolean containsMasterComponent = false; + + private Stack stack; + + private String cardinality = "NOT SPECIFIED"; + + public HostGroupImpl(HostGroupEntity entity, String blueprintName, Stack stack) { + this.name = entity.getName(); + this.cardinality = entity.getCardinality(); + this.blueprintName = blueprintName; + this.stack = stack; + + parseComponents(entity); + parseConfigurations(entity); + } + + public HostGroupImpl(String name, String bpName, Stack stack, Collection<String> components, Configuration configuration, String cardinality) { + this.name = name; + this.blueprintName = bpName; + this.stack = stack; + + // process each component + for (String component : components) { + addComponent(component); + } + this.configuration = configuration; + if (cardinality != null && ! cardinality.equals("null")) { + this.cardinality = cardinality; + } + } + + @Override + public String getName() { + return name; + } + + //todo: currently not qualifying host group name + @Override + public String getFullyQualifiedName() { + return String.format("%s:%s", blueprintName, getName()); + } + + //todo: currently not qualifying host group name + public static String formatAbsoluteName(String bpName, String hgName) { + return String.format("%s:%s", bpName, hgName); + } + + @Override + public Collection<String> getComponents() { + return components; + } + + /** + * Get the services which are deployed to this host group. + * + * @return collection of services which have components in this host group + */ + @Override + public Collection<String> getServices() { + return componentsForService.keySet(); + } + + /** + * Add a component to the host group. + * + * @param component component to add + * + * @return true if component was added; false if component already existed + */ + @Override + public boolean addComponent(String component) { + boolean added = components.add(component); + if (stack.isMasterComponent(component)) { + containsMasterComponent = true; + } + if (added) { + String service = stack.getServiceForComponent(component); + if (service != null) { + // an example of a component without a service in the stack is AMBARI_SERVER + Set<String> serviceComponents = componentsForService.get(service); + if (serviceComponents == null) { + serviceComponents = new HashSet<String>(); + componentsForService.put(service, serviceComponents); + } + serviceComponents.add(component); + } + } + return added; + } + + /** + * Get the components for the specified service which are associated with the host group. + * + * @param service service name + * + * @return set of component names + */ + @Override + public Collection<String> getComponents(String service) { + return componentsForService.containsKey(service) ? + new HashSet<String>(componentsForService.get(service)) : + Collections.<String>emptySet(); + } + + /** + * Get this host groups configuration. + * + * @return configuration instance + */ + @Override + public Configuration getConfiguration() { + + return configuration; + } + + /** + * Get the associated blueprint name. + * + * @return associated blueprint name + */ + @Override + public String getBlueprintName() { + return blueprintName; + } + + @Override + public boolean containsMasterComponent() { + return containsMasterComponent; + } + + @Override + public Stack getStack() { + return stack; + } + + @Override + public String getCardinality() { + return cardinality; + } + + /** + * Parse component information. + */ + private void parseComponents(HostGroupEntity entity) { + for (HostGroupComponentEntity componentEntity : entity.getComponents() ) { + addComponent(componentEntity.getName()); + } + } + + /** + * Parse host group configurations. + */ + //todo: use ConfigurationFactory + private void parseConfigurations(HostGroupEntity entity) { + Map<String, Map<String, String>> config = new HashMap<String, Map<String, String>>(); + Gson jsonSerializer = new Gson(); + for (HostGroupConfigEntity configEntity : entity.getConfigurations()) { + String type = configEntity.getType(); + Map<String, String> typeProperties = config.get(type); + if ( typeProperties == null) { + typeProperties = new HashMap<String, String>(); + config.put(type, typeProperties); + } + Map<String, String> propertyMap = jsonSerializer.<Map<String, String>>fromJson( + configEntity.getConfigData(), Map.class); + + if (propertyMap != null) { + typeProperties.putAll(propertyMap); + } + } + //todo: parse attributes + Map<String, Map<String, Map<String, String>>> attributes = new HashMap<String, Map<String, Map<String, String>>>(); + configuration = new Configuration(config, attributes); + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/c9f0dd0b/ambari-server/src/main/java/org/apache/ambari/server/topology/HostGroupInfo.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/HostGroupInfo.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/HostGroupInfo.java new file mode 100644 index 0000000..07cc1b2 --- /dev/null +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/HostGroupInfo.java @@ -0,0 +1,91 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ambari.server.topology; + +import org.apache.ambari.server.controller.spi.Predicate; + +import java.util.Collection; +import java.util.HashSet; + +/** + * Host Group information specific to a cluster instance. + */ +public class HostGroupInfo { + + private String hostGroupName; + /** + * Hosts contained associated with the host group + */ + private Collection<String> hostNames = new HashSet<String>(); + + private int requested_count = 0; + + Configuration configuration; + + + Predicate predicate; + + + public HostGroupInfo(String hostGroupName) { + this.hostGroupName = hostGroupName; + } + + public String getHostGroupName() { + return hostGroupName; + } + + public Collection<String> getHostNames() { + return new HashSet<String>(hostNames); + } + + public int getRequestedHostCount() { + return requested_count == 0 ? hostNames.size() : requested_count; + } + + public void addHost(String hostName) { + hostNames.add(hostName); + } + + public void addHosts(Collection<String> hosts) { + for (String host : hosts) { + addHost(host); + } + } + + public void setRequestedCount(int num) { + requested_count = num; + } + + //todo: constructor? + public void setConfiguration(Configuration configuration) { + this.configuration = configuration; + } + + public Configuration getConfiguration() { + return configuration; + } + + public void setPredicate(Predicate predicate) { + this.predicate = predicate; + } + + public Predicate getPredicate() { + return predicate; + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/c9f0dd0b/ambari-server/src/main/java/org/apache/ambari/server/topology/HostOfferResponse.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/HostOfferResponse.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/HostOfferResponse.java new file mode 100644 index 0000000..ce636e2 --- /dev/null +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/HostOfferResponse.java @@ -0,0 +1,62 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distribut + * ed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ambari.server.topology; + +import java.util.List; + +/** + * Response to a host offer. + */ +public class HostOfferResponse { + public enum Answer {ACCEPTED, DECLINED_PREDICATE, DECLINED_DONE} + + private final Answer answer; + private final String hostGroupName; + private final List<TopologyTask> tasks; + + public HostOfferResponse(Answer answer) { + if (answer == Answer.ACCEPTED) { + throw new IllegalArgumentException("For accepted response, hostgroup name and tasks must be set"); + } + this.answer = answer; + this.hostGroupName = null; + this.tasks = null; + } + + public HostOfferResponse(Answer answer, String hostGroupName, List<TopologyTask> tasks) { + this.answer = answer; + this.hostGroupName = hostGroupName; + this.tasks = tasks; + } + + public Answer getAnswer() { + return answer; + } + + //todo: for now assumes a host was added + //todo: perhaps a topology modification object that modifies a passed in topology structure? + public String getHostGroupName() { + return hostGroupName; + } + + public List<TopologyTask> getTasks() { + return tasks; + } +}
