Repository: ambari Updated Branches: refs/heads/trunk cc51c2a60 -> 2885f1912
AMBARI-6669. Provide properties_attributes support in cluster creation from blueprint Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/2885f191 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/2885f191 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/2885f191 Branch: refs/heads/trunk Commit: 2885f1912559a2cab1dd0e69495844411425cbdd Parents: cc51c2a Author: Srimanth Gunturi <sgunt...@hortonworks.com> Authored: Tue Jul 29 23:29:13 2014 -0700 Committer: Srimanth Gunturi <sgunt...@hortonworks.com> Committed: Wed Jul 30 12:45:26 2014 -0700 ---------------------------------------------------------------------- .../controller/StackConfigurationResponse.java | 35 +- .../internal/BaseBlueprintProcessor.java | 116 ++++- .../internal/ClusterResourceProvider.java | 70 ++- .../StackConfigurationResourceProvider.java | 17 +- .../ambari/server/state/PropertyInfo.java | 25 +- .../api/util/StackExtensionHelperTest.java | 8 +- .../internal/ClusterResourceProviderTest.java | 473 +++++++++++++++++++ .../StackConfigurationResourceProviderTest.java | 79 +++- .../ambari/server/state/PropertyInfoTest.java | 62 +++ .../services/HDFS/configuration/hdfs-site.xml | 45 ++ 10 files changed, 886 insertions(+), 44 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/2885f191/ambari-server/src/main/java/org/apache/ambari/server/controller/StackConfigurationResponse.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/StackConfigurationResponse.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/StackConfigurationResponse.java index 44454ec..96ca232 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/StackConfigurationResponse.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/StackConfigurationResponse.java @@ -19,6 +19,8 @@ package org.apache.ambari.server.controller; +import java.util.Map; + public class StackConfigurationResponse { /** @@ -27,14 +29,15 @@ public class StackConfigurationResponse { * @param propertyValue Property Value * @param propertyDescription Property Description * @param type Configuration type - * @param isFinal Is property final + * @param propertyAttributes Attributes map */ - public StackConfigurationResponse(String propertyName, String propertyValue, String propertyDescription, String type, Boolean isFinal) { + public StackConfigurationResponse(String propertyName, String propertyValue, String propertyDescription, + String type, Map<String, String> propertyAttributes) { setPropertyName(propertyName); setPropertyValue(propertyValue); setPropertyDescription(propertyDescription); setType(type); - setFinal(isFinal); + setPropertyAttributes(propertyAttributes); } /** @@ -45,18 +48,18 @@ public class StackConfigurationResponse { * @param type Configuration type * @param isRequired Is required to be set * @param propertyType Property Type - * @param isFinal Is property final + * @param propertyAttributes Attributes map */ public StackConfigurationResponse(String propertyName, String propertyValue, String propertyDescription, String type, - Boolean isRequired, String propertyType, Boolean isFinal) { + Boolean isRequired, String propertyType, Map<String, String> propertyAttributes) { setPropertyName(propertyName); setPropertyValue(propertyValue); setPropertyDescription(propertyDescription); setType(type); setRequired(isRequired); setPropertyType(propertyType); - setFinal(isFinal); + setPropertyAttributes(propertyAttributes); } private String stackName; @@ -66,7 +69,7 @@ public class StackConfigurationResponse { private String propertyValue; private String propertyDescription; private String type; - private Boolean isFinal; + private Map<String, String> propertyAttributes; private Boolean isRequired; private String propertyType; @@ -130,12 +133,22 @@ public class StackConfigurationResponse { this.type = type; } - public Boolean isFinal() { - return isFinal; + /** + * Provides attributes of this configuration. + * + * @return Map of attribute name to attribute value + */ + public Map<String, String> getPropertyAttributes() { + return propertyAttributes; } - public void setFinal(Boolean isFinal) { - this.isFinal = isFinal; + /** + * Sets attributes for this configuration. + * + * @param propertyAttributes Map of attribute name to attribute value + */ + public void setPropertyAttributes(Map<String, String> propertyAttributes) { + this.propertyAttributes = propertyAttributes; } /** http://git-wip-us.apache.org/repos/asf/ambari/blob/2885f191/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BaseBlueprintProcessor.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BaseBlueprintProcessor.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BaseBlueprintProcessor.java index 800ff3b..b3a8019 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BaseBlueprintProcessor.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BaseBlueprintProcessor.java @@ -206,6 +206,33 @@ public abstract class BaseBlueprintProcessor extends AbstractControllerResourceP } /** + * Process cluster scoped configuration attributes contained in blueprint. + * + * @param blueprint blueprint entity + * + * @return cluster scoped property attributes contained within in blueprint + */ + protected Map<String, Map<String, Map<String, String>>> processBlueprintAttributes(BlueprintEntity blueprint) { + + Map<String, Map<String, Map<String, String>>> mapAttributes = + new HashMap<String, Map<String, Map<String, String>>>(); + Collection<BlueprintConfigEntity> configs = blueprint.getConfigurations(); + + if (configs != null) { + Gson gson = new Gson(); + for (BlueprintConfigEntity config : configs) { + Map<String, Map<String, String>> typeAttrs = + gson.<Map<String, Map<String, String>>>fromJson(config.getConfigAttributes(), Map.class); + if (typeAttrs != null && !typeAttrs.isEmpty()) { + mapAttributes.put(config.getType(), typeAttrs); + } + } + } + + return mapAttributes; + } + + /** * Override existing properties or add new. * * @param existingProperties current property values @@ -480,8 +507,38 @@ public abstract class BaseBlueprintProcessor extends AbstractControllerResourceP /** * Map of service to config type properties */ - private Map<String, Map<String, Map<String, String>>> serviceConfigurations = - new HashMap<String, Map<String, Map<String, String>>>(); + private Map<String, Map<String, Map<String, ConfigProperty>>> serviceConfigurations = + new HashMap<String, Map<String, Map<String, ConfigProperty>>>(); + + /** + * Contains a configuration property's value and attributes. + */ + private class ConfigProperty { + + private ConfigProperty(String value, Map<String, String> attributes) { + this.value = value; + this.attributes = attributes; + } + + private String value; + private Map<String, String> attributes; + + public String getValue() { + return value; + } + + public void setValue(String value) { + this.value = value; + } + + public Map<String, String> getAttributes() { + return attributes; + } + + public void setAttributes(Map<String, String> attributes) { + this.attributes = attributes; + } + } /** * Constructor. @@ -565,7 +622,47 @@ public abstract class BaseBlueprintProcessor extends AbstractControllerResourceP * @return map of property names to values for the specified service and configuration type */ public Map<String, String> getConfigurationProperties(String service, String type) { - return serviceConfigurations.get(service).get(type); + Map<String, String> configMap = new HashMap<String, String>(); + Map<String, ConfigProperty> configProperties = serviceConfigurations.get(service).get(type); + if (configProperties != null) { + for (Map.Entry<String, ConfigProperty> configProperty : configProperties.entrySet()) { + configMap.put(configProperty.getKey(), configProperty.getValue().getValue()); + } + } + return configMap; + } + + /** + * Get config attributes for the specified service and configuration type. + * + * @param service service name + * @param type configuration type + * + * @return map of attribute names to map of property names to attribute values + * for the specified service and configuration type + */ + public Map<String, Map<String, String>> getConfigurationAttributes(String service, String type) { + Map<String, Map<String, String>> attributesMap = new HashMap<String, Map<String, String>>(); + Map<String, ConfigProperty> configProperties = serviceConfigurations.get(service).get(type); + if (configProperties != null) { + for (Map.Entry<String, ConfigProperty> configProperty : configProperties.entrySet()) { + String propertyName = configProperty.getKey(); + Map<String, String> propertyAttributes = configProperty.getValue().getAttributes(); + if (propertyAttributes != null) { + for (Map.Entry<String, String> propertyAttribute : propertyAttributes.entrySet()) { + String attributeName = propertyAttribute.getKey(); + String attributeValue = propertyAttribute.getValue(); + Map<String, String> attributes = attributesMap.get(attributeName); + if (attributes == null) { + attributes = new HashMap<String, String>(); + attributesMap.put(attributeName, attributes); + } + attributes.put(propertyName, attributeValue); + } + } + } + } + return attributesMap; } /** @@ -603,8 +700,8 @@ public abstract class BaseBlueprintProcessor extends AbstractControllerResourceP * @return name of service which corresponds to the specified configuration type */ public String getServiceForConfigType(String config) { - for (Map.Entry<String, Map<String, Map<String, String>>> entry : serviceConfigurations.entrySet()) { - Map<String, Map<String, String>> typeMap = entry.getValue(); + for (Map.Entry<String, Map<String, Map<String, ConfigProperty>>> entry : serviceConfigurations.entrySet()) { + Map<String, Map<String, ConfigProperty>> typeMap = entry.getValue(); if (typeMap.containsKey(config)) { return entry.getKey(); } @@ -702,7 +799,7 @@ public abstract class BaseBlueprintProcessor extends AbstractControllerResourceP * @throws AmbariException an exception occurred getting configurations from the stack definition */ private void parseConfigurations(String service) throws AmbariException { - Map<String, Map<String, String>> mapServiceConfig = new HashMap<String, Map<String, String>>(); + Map<String, Map<String, ConfigProperty>> mapServiceConfig = new HashMap<String, Map<String, ConfigProperty>>(); serviceConfigurations.put(service, mapServiceConfig); @@ -715,12 +812,13 @@ public abstract class BaseBlueprintProcessor extends AbstractControllerResourceP if (type.endsWith(".xml")) { type = type.substring(0, type.length() - 4); } - Map<String, String> mapTypeConfig = mapServiceConfig.get(type); + Map<String, ConfigProperty> mapTypeConfig = mapServiceConfig.get(type); if (mapTypeConfig == null) { - mapTypeConfig = new HashMap<String, String>(); + mapTypeConfig = new HashMap<String, ConfigProperty>(); mapServiceConfig.put(type, mapTypeConfig); } - mapTypeConfig.put(config.getPropertyName(), config.getPropertyValue()); + mapTypeConfig.put(config.getPropertyName(), + new ConfigProperty(config.getPropertyValue(), config.getPropertyAttributes())); } } http://git-wip-us.apache.org/repos/asf/ambari/blob/2885f191/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterResourceProvider.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterResourceProvider.java index 464f116..a39e5ad 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterResourceProvider.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterResourceProvider.java @@ -84,6 +84,11 @@ public class ClusterResourceProvider extends BaseBlueprintProcessor { */ private Map<String, Map<String, String>> mapClusterConfigurations = new HashMap<String, Map<String, String>>(); + /** + * Maps configuration type (string) to property attributes, and their values + */ + private Map<String, Map<String, Map<String, String>>> mapClusterAttributes = + new HashMap<String, Map<String, Map<String, String>>>(); // ----- Constructors ---------------------------------------------------- @@ -314,8 +319,9 @@ public class ClusterResourceProvider extends BaseBlueprintProcessor { Map<String, HostGroup> blueprintHostGroups = parseBlueprintHostGroups(blueprint, stack); applyRequestInfoToHostGroups(properties, blueprintHostGroups); - processConfigurations(processBlueprintConfigurations(blueprint, (Collection<Map<String, String>>) - properties.get("configurations")), stack, blueprintHostGroups); + Collection<Map<String, String>> configOverrides = (Collection<Map<String, String>>)properties.get("configurations"); + processConfigurations(processBlueprintConfigurations(blueprint, configOverrides), + processBlueprintAttributes(blueprint), stack, blueprintHostGroups); validatePasswordProperties(blueprint, blueprintHostGroups, (String) properties.get("default_password")); String clusterName = (String) properties.get(CLUSTER_NAME_PROPERTY_ID); @@ -573,9 +579,10 @@ public class ClusterResourceProvider extends BaseBlueprintProcessor { for (Map.Entry<String, Map<String, String>> entry : mapClusterConfigurations.entrySet()) { String type = entry.getKey(); + Map<String, Map<String, String>> confAttributes = mapClusterAttributes.get(type); try { //todo: properly handle non system exceptions - setConfigurationsOnCluster(clusterName, type, entry.getValue()); + setConfigurationsOnCluster(clusterName, type, entry.getValue(), confAttributes); } catch (AmbariException e) { throw new SystemException("Unable to set configurations on cluster.", e); } @@ -592,7 +599,8 @@ public class ClusterResourceProvider extends BaseBlueprintProcessor { * @throws AmbariException if an exception occurs setting the properties */ private void setConfigurationsOnCluster(String clusterName, String type, - Map<String, String> properties) throws AmbariException { + Map<String, String> properties, + Map<String, Map<String, String>> propertiesAttributes) throws AmbariException { Map<String, Object> clusterProperties = new HashMap<String, Object>(); clusterProperties.put(CLUSTER_NAME_PROPERTY_ID, clusterName); @@ -602,6 +610,15 @@ public class ClusterResourceProvider extends BaseBlueprintProcessor { clusterProperties.put(CLUSTER_DESIRED_CONFIGS_PROPERTY_ID + "/properties/" + entry.getKey(), entry.getValue()); } + if (propertiesAttributes != null) { + for (Map.Entry<String, Map<String, String>> attribute : propertiesAttributes.entrySet()) { + String attributeName = attribute.getKey(); + for (Map.Entry<String, String> attributeOccurrence : attribute.getValue().entrySet()) { + clusterProperties.put(CLUSTER_DESIRED_CONFIGS_PROPERTY_ID + "/properties_attributes/" + + attributeName + "/" + attributeOccurrence.getKey(), attributeOccurrence.getValue()); + } + } + } getManagementController().updateClusters( Collections.singleton(getRequest(clusterProperties)), null); } @@ -708,7 +725,8 @@ public class ClusterResourceProvider extends BaseBlueprintProcessor { * @param blueprintHostGroups host groups contained in the blueprint */ private void processConfigurations(Map<String, Map<String, String>> blueprintConfigurations, - Stack stack, Map<String, HostGroup> blueprintHostGroups) { + Map<String, Map<String, Map<String, String>>> blueprintAttributes, + Stack stack, Map<String, HostGroup> blueprintHostGroups) { for (String service : getServicesToDeploy(stack, blueprintHostGroups)) { for (String type : stack.getConfigurationTypes(service)) { @@ -718,9 +736,26 @@ public class ClusterResourceProvider extends BaseBlueprintProcessor { mapClusterConfigurations.put(type, typeProps); } typeProps.putAll(stack.getConfigurationProperties(service, type)); + Map<String, Map<String, String>> stackTypeAttributes = stack.getConfigurationAttributes(service, type); + if (!stackTypeAttributes.isEmpty()) { + if (!mapClusterAttributes.containsKey(type)) { + mapClusterAttributes.put(type, new HashMap<String, Map<String, String>>()); + } + Map<String, Map<String, String>> typeAttrs = mapClusterAttributes.get(type); + for (Map.Entry<String, Map<String, String>> attribute : stackTypeAttributes.entrySet()) { + String attributeName = attribute.getKey(); + Map<String, String> attributes = typeAttrs.get(attributeName); + if (attributes == null) { + attributes = new HashMap<String, String>(); + typeAttrs.put(attributeName, attributes); + } + attributes.putAll(attribute.getValue()); + } + } } } processBlueprintClusterConfigurations(blueprintConfigurations); + processBlueprintClusterConfigAttributes(blueprintAttributes); for (Map.Entry<String, Map<String, String>> entry : mapClusterConfigurations.entrySet()) { for (Map.Entry<String, String> propertyEntry : entry.getValue().entrySet()) { @@ -757,6 +792,31 @@ public class ClusterResourceProvider extends BaseBlueprintProcessor { } /** + * Process cluster scoped configuration attributes provided in blueprint. + * + * @param blueprintAttributes map of configuration type to configuration attributes and their values + */ + private void processBlueprintClusterConfigAttributes(Map<String, Map<String, Map<String, String>>> blueprintAttributes) { + for (Map.Entry<String, Map<String, Map<String, String>>> entry : blueprintAttributes.entrySet()) { + Map<String, Map<String, String>> attributes = entry.getValue(); + if (attributes != null && !attributes.isEmpty()) { + String type = entry.getKey(); + if (!mapClusterAttributes.containsKey(type)) { + mapClusterAttributes.put(type, new HashMap<String, Map<String, String>>()); + } + Map<String, Map<String, String>> typeAttrs = mapClusterAttributes.get(type); + for (Map.Entry<String, Map<String, String>> attribute : attributes.entrySet()) { + String attributeName = attribute.getKey(); + if (!typeAttrs.containsKey(attributeName)) { + typeAttrs.put(attributeName, new HashMap<String, String>()); + } + typeAttrs.get(attributeName).putAll(attribute.getValue()); + } + } + } + } + + /** * Explicitly set any properties that are required but not currently provided in the stack definition. */ private void setMissingConfigurations() { http://git-wip-us.apache.org/repos/asf/ambari/blob/2885f191/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackConfigurationResourceProvider.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackConfigurationResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackConfigurationResourceProvider.java index 4aa56f0..2e2bfe1 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackConfigurationResourceProvider.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackConfigurationResourceProvider.java @@ -128,8 +128,12 @@ public class StackConfigurationResourceProvider extends setResourceProperty(resource, PROPERTY_TYPE_PROPERTY_ID, response.getType(), requestedIds); - setResourceProperty(resource, PROPERTY_FINAL_PROPERTY_ID, - response.isFinal(), requestedIds); + setDefaultPropertiesAttributes(resource, requestedIds); + + for (Map.Entry<String, String> attribute : response.getPropertyAttributes().entrySet()) { + setResourceProperty(resource, PropertyHelper.getPropertyId("StackConfigurations", attribute.getKey()), + attribute.getValue(), requestedIds); + } resources.add(resource); } @@ -137,6 +141,15 @@ public class StackConfigurationResourceProvider extends return resources; } + /** + * Set default values for properties attributes before applying original ones + * to prevent absence in case of empty attributes map + */ + private void setDefaultPropertiesAttributes(Resource resource, Set<String> requestedIds) { + setResourceProperty(resource, PROPERTY_FINAL_PROPERTY_ID, + "false", requestedIds); + } + private StackConfigurationRequest getRequest(Map<String, Object> properties) { return new StackConfigurationRequest( (String) properties.get(STACK_NAME_PROPERTY_ID), http://git-wip-us.apache.org/repos/asf/ambari/blob/2885f191/ambari-server/src/main/java/org/apache/ambari/server/state/PropertyInfo.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/PropertyInfo.java b/ambari-server/src/main/java/org/apache/ambari/server/state/PropertyInfo.java index 8789383..6a17e9d 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/PropertyInfo.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/PropertyInfo.java @@ -20,9 +20,14 @@ package org.apache.ambari.server.state; import org.apache.ambari.server.controller.StackConfigurationResponse; +import org.w3c.dom.Element; +import javax.xml.bind.annotation.XmlAnyElement; import javax.xml.bind.annotation.XmlAttribute; -import javax.xml.bind.annotation.XmlElement; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; public class PropertyInfo { private String name; @@ -32,7 +37,8 @@ public class PropertyInfo { private boolean deleted; private boolean requireInput; private PropertyType type = PropertyType.DEFAULT; - private boolean isFinal; + @XmlAnyElement + private List<Element> propertyAttributes = new ArrayList<Element>(); public String getName() { return name; @@ -68,7 +74,7 @@ public class PropertyInfo { public StackConfigurationResponse convertToResponse() { return new StackConfigurationResponse(getName(), getValue(), - getDescription() , getFilename(), isRequireInput(), getType().name(), isFinal()); + getDescription() , getFilename(), isRequireInput(), getType().name(), getAttributesMap()); } public boolean isDeleted() { @@ -79,13 +85,12 @@ public class PropertyInfo { this.deleted = deleted; } - @XmlElement(name="final") - public boolean isFinal() { - return isFinal; - } - - public void setFinal(boolean isFinal) { - this.isFinal = isFinal; + public Map<String, String> getAttributesMap() { + Map<String, String> attributes = new HashMap<String, String>(); + for (Element propertyAttribute : propertyAttributes) { + attributes.put(propertyAttribute.getTagName(), propertyAttribute.getFirstChild().getNodeValue()); + } + return attributes; } @XmlAttribute(name = "require-input") http://git-wip-us.apache.org/repos/asf/ambari/blob/2885f191/ambari-server/src/test/java/org/apache/ambari/server/api/util/StackExtensionHelperTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/api/util/StackExtensionHelperTest.java b/ambari-server/src/test/java/org/apache/ambari/server/api/util/StackExtensionHelperTest.java index 7d07fe1..517f3b7 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/api/util/StackExtensionHelperTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/api/util/StackExtensionHelperTest.java @@ -599,7 +599,8 @@ public class StackExtensionHelperTest { assertEquals("yarn.scheduler.capacity.maximum-applications", propertyInfo.getName()); assertEquals("Maximum number of applications that can be pending and running.", propertyInfo.getDescription()); assertEquals("10000", propertyInfo.getValue()); - assertEquals(true, propertyInfo.isFinal()); + assertEquals(1, propertyInfo.getAttributesMap().size()); + assertEquals("true", propertyInfo.getAttributesMap().get("final")); assertEquals(null, propertyInfo.getFilename()); assertEquals(false, propertyInfo.isDeleted()); assertEquals(false, propertyInfo.isRequireInput()); @@ -609,7 +610,8 @@ public class StackExtensionHelperTest { assertEquals("yarn.scheduler.capacity.maximum-am-resource-percent", propertyInfo.getName()); assertEquals("Maximum percent of resources in the cluster.", propertyInfo.getDescription()); assertEquals("0.2", propertyInfo.getValue()); - assertEquals(false, propertyInfo.isFinal()); + assertEquals(1, propertyInfo.getAttributesMap().size()); + assertEquals("false", propertyInfo.getAttributesMap().get("final")); assertEquals(null, propertyInfo.getFilename()); assertEquals(true, propertyInfo.isDeleted()); assertEquals(false, propertyInfo.isRequireInput()); @@ -619,7 +621,7 @@ public class StackExtensionHelperTest { assertEquals("yarn.scheduler.capacity.root.queues", propertyInfo.getName()); assertEquals("The queues at the this level (root is the root queue).", propertyInfo.getDescription()); assertEquals("default", propertyInfo.getValue()); - assertEquals(false, propertyInfo.isFinal()); + assertEquals(0, propertyInfo.getAttributesMap().size()); assertEquals(null, propertyInfo.getFilename()); assertEquals(false, propertyInfo.isDeleted()); assertEquals(true, propertyInfo.isRequireInput()); http://git-wip-us.apache.org/repos/asf/ambari/blob/2885f191/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ClusterResourceProviderTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ClusterResourceProviderTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ClusterResourceProviderTest.java index 11e5993..a1cb64e 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ClusterResourceProviderTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ClusterResourceProviderTest.java @@ -28,6 +28,7 @@ import static org.easymock.EasyMock.expect; import static org.easymock.EasyMock.replay; import static org.easymock.EasyMock.verify; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -187,6 +188,11 @@ public class ClusterResourceProviderTest { Capture<Set<StackConfigurationRequest>> serviceConfigurationRequestCapture1 = new Capture<Set<StackConfigurationRequest>>(); Capture<Set<StackConfigurationRequest>> serviceConfigurationRequestCapture2 = new Capture<Set<StackConfigurationRequest>>(); + Map<String, String> coreSiteAttributes = new HashMap<String, String>(); + coreSiteAttributes.put("final", "true"); + Map<String, String> hdfsSiteAttributes = new HashMap<String, String>(); + hdfsSiteAttributes.put("final", "true"); + BlueprintConfigEntity blueprintConfig = createNiceMock(BlueprintConfigEntity.class); BlueprintConfigEntity blueprintConfig2 = createNiceMock(BlueprintConfigEntity.class); @@ -322,6 +328,7 @@ public class ClusterResourceProviderTest { expect(stackConfigurationResponse1.getType()).andReturn("core-site.xml"); expect(stackConfigurationResponse1.getPropertyName()).andReturn("property1"); expect(stackConfigurationResponse1.getPropertyValue()).andReturn("value1"); + expect(stackConfigurationResponse1.getPropertyAttributes()).andReturn(coreSiteAttributes); expect(managementController.getStackComponents(capture(serviceComponentRequestCapture2))). andReturn(stackServiceComponentResponses2); @@ -332,6 +339,7 @@ public class ClusterResourceProviderTest { expect(stackConfigurationResponse2.getType()).andReturn("hdfs-site.xml"); expect(stackConfigurationResponse2.getPropertyName()).andReturn("property2"); expect(stackConfigurationResponse2.getPropertyValue()).andReturn("value2"); + expect(stackConfigurationResponse2.getPropertyAttributes()).andReturn(hdfsSiteAttributes); expect(stackConfigurationResponse3.getType()).andReturn("global.xml"); expect(stackConfigurationResponse3.getPropertyName()).andReturn("oozie_user"); @@ -500,6 +508,10 @@ public class ClusterResourceProviderTest { ConfigurationRequest hdfsConfigRequest = mapConfigRequests.get("hdfs-site"); assertEquals(1, hdfsConfigRequest.getProperties().size()); assertEquals("value2", hdfsConfigRequest.getProperties().get("property2")); + Map<String, Map<String, String>> hdfsAttributes = hdfsConfigRequest.getPropertiesAttributes(); + assertTrue(hdfsAttributes.containsKey("final")); + assertEquals(1, hdfsAttributes.get("final").size()); + assertEquals("true", hdfsAttributes.get("final").get("property2")); ConfigurationRequest coreConfigRequest = mapConfigRequests.get("core-site"); assertEquals(5, coreConfigRequest.getProperties().size()); assertEquals("value2", coreConfigRequest.getProperties().get("property1")); @@ -507,6 +519,10 @@ public class ClusterResourceProviderTest { assertEquals("*", coreConfigRequest.getProperties().get("hadoop.proxyuser.oozie.hosts")); assertEquals("users", coreConfigRequest.getProperties().get("hadoop.proxyuser.oozie.groups")); assertEquals("new.property.value", coreConfigRequest.getProperties().get("new.property")); + Map<String, Map<String, String>> coreAttributes = coreConfigRequest.getPropertiesAttributes(); + assertTrue(coreAttributes.containsKey("final")); + assertEquals(1, coreAttributes.get("final").size()); + assertEquals("true", coreAttributes.get("final").get("property1")); ConfigurationRequest hiveConfigRequest = mapConfigRequests.get("hive-site"); assertEquals(1, hiveConfigRequest.getProperties().size()); assertEquals("host.domain:12345", hiveConfigRequest.getProperties().get("javax.jdo.option.ConnectionURL")); @@ -1904,6 +1920,463 @@ public class ClusterResourceProviderTest { } } + @Test + public void testCreateResource_blueprint_attrbiutesProvided() throws Exception { + String blueprintName = "test-blueprint"; + String stackName = "test"; + String stackVersion = "1.23"; + String clusterName = "c1"; + + BlueprintDAO blueprintDAO = createStrictMock(BlueprintDAO.class); + AmbariMetaInfo metaInfo = createMock(AmbariMetaInfo.class); + AmbariManagementController managementController = createStrictMock(AmbariManagementController.class); + Request request = createNiceMock(Request.class); + RequestStatusResponse response = createNiceMock(RequestStatusResponse.class); + BlueprintEntity blueprint = createNiceMock(BlueprintEntity.class); + StackServiceResponse stackServiceResponse1 = createNiceMock(StackServiceResponse.class); + StackServiceResponse stackServiceResponse2 = createNiceMock(StackServiceResponse.class); + Capture<Set<StackServiceRequest>> stackServiceRequestCapture = new Capture<Set<StackServiceRequest>>(); + + StackServiceComponentResponse stackServiceComponentResponse1 = createNiceMock(StackServiceComponentResponse.class); + StackServiceComponentResponse stackServiceComponentResponse2 = createNiceMock(StackServiceComponentResponse.class); + StackServiceComponentResponse stackServiceComponentResponse3 = createNiceMock(StackServiceComponentResponse.class); + StackServiceComponentResponse stackServiceComponentResponse4 = createNiceMock(StackServiceComponentResponse.class); + Capture<Set<StackServiceComponentRequest>> serviceComponentRequestCapture1 = new Capture<Set<StackServiceComponentRequest>>(); + Capture<Set<StackServiceComponentRequest>> serviceComponentRequestCapture2 = new Capture<Set<StackServiceComponentRequest>>(); + + StackConfigurationResponse stackConfigurationResponse1 = createNiceMock(StackConfigurationResponse.class); + StackConfigurationResponse stackConfigurationResponse2 = createNiceMock(StackConfigurationResponse.class); + StackConfigurationResponse stackConfigurationResponse3 = createNiceMock(StackConfigurationResponse.class); + StackConfigurationResponse stackConfigurationResponse4 = createNiceMock(StackConfigurationResponse.class); + StackConfigurationResponse stackConfigurationResponse5 = createNiceMock(StackConfigurationResponse.class); + Capture<Set<StackConfigurationRequest>> serviceConfigurationRequestCapture1 = new Capture<Set<StackConfigurationRequest>>(); + Capture<Set<StackConfigurationRequest>> serviceConfigurationRequestCapture2 = new Capture<Set<StackConfigurationRequest>>(); + + Map<String, String> coreSiteAttributes = new HashMap<String, String>(); + coreSiteAttributes.put("final", "true"); + Map<String, String> hdfsSiteAttributes = new HashMap<String, String>(); + hdfsSiteAttributes.put("final", "true"); + + BlueprintConfigEntity blueprintConfig = createNiceMock(BlueprintConfigEntity.class); + BlueprintConfigEntity blueprintConfig2 = createNiceMock(BlueprintConfigEntity.class); + + HostGroupEntity hostGroup = createNiceMock(HostGroupEntity.class); + HostGroupComponentEntity hostGroupComponent1 = createNiceMock(HostGroupComponentEntity.class); + HostGroupComponentEntity hostGroupComponent2 = createNiceMock(HostGroupComponentEntity.class); + HostGroupComponentEntity hostGroupComponent3 = createNiceMock(HostGroupComponentEntity.class); + HostGroupComponentEntity hostGroupComponent4 = createNiceMock(HostGroupComponentEntity.class); + + HostGroupConfigEntity hostGroupConfig = createNiceMock(HostGroupConfigEntity.class); + + ServiceResourceProvider serviceResourceProvider = createStrictMock(ServiceResourceProvider.class); + ResourceProvider componentResourceProvider = createStrictMock(ResourceProvider.class); + ResourceProvider hostResourceProvider = createStrictMock(ResourceProvider.class); + ResourceProvider hostComponentResourceProvider = createStrictMock(ResourceProvider.class); + ConfigGroupResourceProvider configGroupResourceProvider = createStrictMock(ConfigGroupResourceProvider.class); + PersistKeyValueImpl persistKeyValue = createNiceMock(PersistKeyValueImpl.class); + + Capture<ClusterRequest> createClusterRequestCapture = new Capture<ClusterRequest>(); + Capture<Set<ClusterRequest>> updateClusterRequestCapture = new Capture<Set<ClusterRequest>>(); + Capture<Map<String, String>> updateClusterPropertyMapCapture = new Capture<Map<String, String>>(); + Capture<Set<ClusterRequest>> updateClusterRequestCapture2 = new Capture<Set<ClusterRequest>>(); + Capture<Map<String, String>> updateClusterPropertyMapCapture2 = new Capture<Map<String, String>>(); + Capture<Set<ClusterRequest>> updateClusterRequestCapture3 = new Capture<Set<ClusterRequest>>(); + Capture<Map<String, String>> updateClusterPropertyMapCapture3 = new Capture<Map<String, String>>(); + Capture<Set<ClusterRequest>> updateClusterRequestCapture4 = new Capture<Set<ClusterRequest>>(); + Capture<Map<String, String>> updateClusterPropertyMapCapture4 = new Capture<Map<String, String>>(); + Capture<Set<ClusterRequest>> updateClusterRequestCapture5 = new Capture<Set<ClusterRequest>>(); + Capture<Map<String, String>> updateClusterPropertyMapCapture5 = new Capture<Map<String, String>>(); + + Capture<Request> serviceRequestCapture = new Capture<Request>(); + Capture<Request> componentRequestCapture = new Capture<Request>(); + Capture<Request> componentRequestCapture2 = new Capture<Request>(); + Capture<Request> hostRequestCapture = new Capture<Request>(); + Capture<Request> hostComponentRequestCapture = new Capture<Request>(); + Capture<Set<ConfigGroupRequest>> configGroupRequestCapture = new Capture<Set<ConfigGroupRequest>>(); + + Set<StackServiceResponse> stackServiceResponses = new LinkedHashSet<StackServiceResponse>(); + stackServiceResponses.add(stackServiceResponse1); + stackServiceResponses.add(stackServiceResponse2); + + // service1 has 3 components + Set<StackServiceComponentResponse> stackServiceComponentResponses1 = new LinkedHashSet<StackServiceComponentResponse>(); + stackServiceComponentResponses1.add(stackServiceComponentResponse1); + stackServiceComponentResponses1.add(stackServiceComponentResponse2); + stackServiceComponentResponses1.add(stackServiceComponentResponse4); + + // service2 has 1 components + Set<StackServiceComponentResponse> stackServiceComponentResponses2 = new LinkedHashSet<StackServiceComponentResponse>(); + stackServiceComponentResponses2.add(stackServiceComponentResponse3); + + // service1 has 2 config + Set<StackConfigurationResponse> stackConfigurationResponses1 = new LinkedHashSet<StackConfigurationResponse>(); + stackConfigurationResponses1.add(stackConfigurationResponse1); + stackConfigurationResponses1.add(stackConfigurationResponse5); + + // service2 has 3 config + Set<StackConfigurationResponse> stackConfigurationResponses2 = new LinkedHashSet<StackConfigurationResponse>(); + stackConfigurationResponses2.add(stackConfigurationResponse2); + stackConfigurationResponses2.add(stackConfigurationResponse3); + stackConfigurationResponses2.add(stackConfigurationResponse4); + + Collection<HostGroupComponentEntity> hostGroupComponents = new LinkedHashSet<HostGroupComponentEntity>(); + hostGroupComponents.add(hostGroupComponent1); + hostGroupComponents.add(hostGroupComponent2); + hostGroupComponents.add(hostGroupComponent3); + hostGroupComponents.add(hostGroupComponent4); + + // request properties + Set<Map<String, Object>> propertySet = new LinkedHashSet<Map<String, Object>>(); + Map<String, Object> properties = new LinkedHashMap<String, Object>(); + + properties.put(ClusterResourceProvider.CLUSTER_NAME_PROPERTY_ID, clusterName); + properties.put(ClusterResourceProvider.BLUEPRINT_PROPERTY_ID, blueprintName); + propertySet.add(properties); + + Collection<Map<String, Object>> hostGroups = new ArrayList<Map<String, Object>>(); + Map<String, Object> hostGroupProperties = new HashMap<String, Object>(); + hostGroups.add(hostGroupProperties); + hostGroupProperties.put("name", "group1"); + Collection<Map<String, String>> hostGroupHosts = new ArrayList<Map<String, String>>(); + hostGroupProperties.put("hosts", hostGroupHosts); + Map<String, String> hostGroupHostProperties = new HashMap<String, String>(); + hostGroupHostProperties.put("fqdn", "host.domain"); + hostGroupHosts.add(hostGroupHostProperties); + properties.put("host_groups", hostGroups); + + Map<String, String> mapGroupConfigProperties = new HashMap<String, String>(); + mapGroupConfigProperties.put("myGroupProp", "awesomeValue"); + + // blueprint core-site cluster configuration properties + Map<String, String> blueprintCoreConfigProperties = new HashMap<String, String>(); + blueprintCoreConfigProperties.put("property1", "value2"); + blueprintCoreConfigProperties.put("new.property", "new.property.value"); + + Map<String, String> blueprintGlobalConfigProperties = new HashMap<String, String>(); + blueprintGlobalConfigProperties.put("hive_database", "New MySQL Database"); + + // blueprint core-site cluster configuration properties + Map<String, Map<String, String>> blueprintCoreConfigAttributes = new HashMap<String, Map<String, String>>(); + Map<String, String> coreFinalAttributes = new HashMap<String, String>(); + coreFinalAttributes.put("property1", "false"); + coreFinalAttributes.put("new.property", "true"); + blueprintCoreConfigAttributes.put("final", coreFinalAttributes); + + Map<String, Map<String, String>> blueprintHiveEnvConfigAttributes = new HashMap<String, Map<String, String>>(); + Map<String, String> globalFinalAttributes = new HashMap<String, String>(); + globalFinalAttributes.put("hive_database", "true"); + blueprintHiveEnvConfigAttributes.put("final", globalFinalAttributes); + + Collection<BlueprintConfigEntity> configurations = new HashSet<BlueprintConfigEntity>(); + configurations.add(blueprintConfig); + configurations.add(blueprintConfig2); + + // expectations + expect(request.getProperties()).andReturn(propertySet).anyTimes(); + expect(blueprintDAO.findByName(blueprintName)).andReturn(blueprint); + expect(blueprint.getStackName()).andReturn(stackName); + expect(blueprint.getStackVersion()).andReturn(stackVersion); + expect(blueprint.getConfigurations()).andReturn(configurations).times(2); + expect(blueprint.validateConfigurations(metaInfo, PropertyInfo.PropertyType.PASSWORD)).andReturn( + Collections.<String, Map<String, Collection<String>>>emptyMap()); + + expect(metaInfo.getComponentDependencies("test", "1.23", "service1", "component1")). + andReturn(Collections.<DependencyInfo>emptyList()); + expect(metaInfo.getComponentDependencies("test", "1.23", "service1", "component2")). + andReturn(Collections.<DependencyInfo>emptyList()); + expect(metaInfo.getComponentDependencies("test", "1.23", "service1", "MYSQL_SERVER")). + andReturn(Collections.<DependencyInfo>emptyList()); + expect(metaInfo.getComponentDependencies("test", "1.23", "service2", "component3")). + andReturn(Collections.<DependencyInfo>emptyList()); + + expect(managementController.getStackServices(capture(stackServiceRequestCapture))).andReturn(stackServiceResponses); + expect(stackServiceResponse1.getServiceName()).andReturn("service1"); + expect(stackServiceResponse2.getServiceName()).andReturn("service2"); + + expect(managementController.getStackComponents(capture(serviceComponentRequestCapture1))). + andReturn(stackServiceComponentResponses1); + expect(stackServiceComponentResponse1.getComponentName()).andReturn("component1"); + expect(stackServiceComponentResponse2.getComponentName()).andReturn("component2"); + expect(stackServiceComponentResponse4.getComponentName()).andReturn("MYSQL_SERVER"); + + expect(managementController.getStackConfigurations(capture(serviceConfigurationRequestCapture1))). + andReturn(stackConfigurationResponses1); + expect(stackConfigurationResponse1.getType()).andReturn("core-site.xml"); + expect(stackConfigurationResponse1.getPropertyName()).andReturn("property1"); + expect(stackConfigurationResponse1.getPropertyValue()).andReturn("value1"); + expect(stackConfigurationResponse1.getPropertyAttributes()).andReturn(coreSiteAttributes); + + expect(managementController.getStackComponents(capture(serviceComponentRequestCapture2))). + andReturn(stackServiceComponentResponses2); + expect(stackServiceComponentResponse3.getComponentName()).andReturn("component3"); + + expect(managementController.getStackConfigurations(capture(serviceConfigurationRequestCapture2))). + andReturn(stackConfigurationResponses2); + expect(stackConfigurationResponse2.getType()).andReturn("hdfs-site.xml"); + expect(stackConfigurationResponse2.getPropertyName()).andReturn("property2"); + expect(stackConfigurationResponse2.getPropertyValue()).andReturn("value2"); + expect(stackConfigurationResponse2.getPropertyAttributes()).andReturn(hdfsSiteAttributes); + + expect(stackConfigurationResponse3.getType()).andReturn("global.xml"); + expect(stackConfigurationResponse3.getPropertyName()).andReturn("oozie_user"); + expect(stackConfigurationResponse3.getPropertyValue()).andReturn("oozie"); + + expect(stackConfigurationResponse4.getType()).andReturn("core-site.xml"); + expect(stackConfigurationResponse4.getPropertyName()).andReturn("property3"); + expect(stackConfigurationResponse4.getPropertyValue()).andReturn("value3"); + + expect(stackConfigurationResponse5.getType()).andReturn("hive-site.xml"); + expect(stackConfigurationResponse5.getPropertyName()).andReturn("javax.jdo.option.ConnectionURL"); + expect(stackConfigurationResponse5.getPropertyValue()).andReturn("localhost:12345"); + + expect(blueprintConfig.getBlueprintName()).andReturn("test-blueprint").anyTimes(); + expect(blueprintConfig.getType()).andReturn("core-site").anyTimes(); + expect(blueprintConfig.getConfigData()).andReturn(new Gson().toJson(blueprintCoreConfigProperties)).anyTimes(); + expect(blueprintConfig.getConfigAttributes()).andReturn(new Gson().toJson(blueprintCoreConfigAttributes)).anyTimes(); + expect(blueprintConfig2.getBlueprintName()).andReturn("test-blueprint").anyTimes(); + expect(blueprintConfig2.getType()).andReturn("hive-env").anyTimes(); + expect(blueprintConfig2.getConfigData()).andReturn(new Gson().toJson(blueprintGlobalConfigProperties)).anyTimes(); + expect(blueprintConfig2.getConfigAttributes()).andReturn(new Gson().toJson(blueprintHiveEnvConfigAttributes)).anyTimes(); + + expect(blueprint.getHostGroups()).andReturn(Collections.singleton(hostGroup)).anyTimes(); + expect(hostGroup.getName()).andReturn("group1").anyTimes(); + expect(hostGroup.getComponents()).andReturn(hostGroupComponents).anyTimes(); + expect(hostGroupComponent1.getName()).andReturn("component1").anyTimes(); + expect(hostGroupComponent2.getName()).andReturn("component2").anyTimes(); + expect(hostGroupComponent3.getName()).andReturn("component3").anyTimes(); + expect(hostGroupComponent4.getName()).andReturn("MYSQL_SERVER").anyTimes(); + expect(hostGroup.getConfigurations()).andReturn( + Collections.<HostGroupConfigEntity>singleton(hostGroupConfig)).anyTimes(); + + expect(hostGroupConfig.getType()).andReturn("core-site").anyTimes(); + expect(hostGroupConfig.getConfigData()).andReturn(new Gson().toJson(mapGroupConfigProperties)).anyTimes(); + + managementController.createCluster(capture(createClusterRequestCapture)); + expect(managementController.updateClusters(capture(updateClusterRequestCapture), + capture(updateClusterPropertyMapCapture))).andReturn(null); + expect(managementController.updateClusters(capture(updateClusterRequestCapture2), + capture(updateClusterPropertyMapCapture2))).andReturn(null); + expect(managementController.updateClusters(capture(updateClusterRequestCapture3), + capture(updateClusterPropertyMapCapture3))).andReturn(null); + expect(managementController.updateClusters(capture(updateClusterRequestCapture4), + capture(updateClusterPropertyMapCapture4))).andReturn(null); + expect(managementController.updateClusters(capture(updateClusterRequestCapture5), + capture(updateClusterPropertyMapCapture5))).andReturn(null); + + expect(serviceResourceProvider.createResources(capture(serviceRequestCapture))).andReturn(null); + expect(componentResourceProvider.createResources(capture(componentRequestCapture))).andReturn(null); + expect(componentResourceProvider.createResources(capture(componentRequestCapture2))).andReturn(null); + expect(hostResourceProvider.createResources(capture(hostRequestCapture))).andReturn(null); + expect(hostComponentResourceProvider.createResources(capture(hostComponentRequestCapture))).andReturn(null); + + expect(serviceResourceProvider.installAndStart(clusterName)).andReturn(response); + + expect(configGroupResourceProvider.createResources( + capture(configGroupRequestCapture))).andReturn(null); + + persistKeyValue.put("CLUSTER_CURRENT_STATUS", "{\"clusterState\":\"CLUSTER_STARTED_5\"}"); + + replay(blueprintDAO, managementController, request, response, blueprint, stackServiceResponse1, stackServiceResponse2, + stackServiceComponentResponse1, stackServiceComponentResponse2, stackServiceComponentResponse3, + stackServiceComponentResponse4, stackConfigurationResponse1, stackConfigurationResponse2, + stackConfigurationResponse3, stackConfigurationResponse4, stackConfigurationResponse5, blueprintConfig, + blueprintConfig2, hostGroup, hostGroupComponent1, hostGroupComponent2, hostGroupComponent3, hostGroupComponent4, + hostGroupConfig, serviceResourceProvider, componentResourceProvider, hostResourceProvider, + hostComponentResourceProvider, configGroupResourceProvider, persistKeyValue, metaInfo); + + // test + ClusterResourceProvider.init(blueprintDAO, metaInfo); + PersistKeyValueService.init(persistKeyValue); + ResourceProvider provider = new TestClusterResourceProvider( + managementController, serviceResourceProvider, componentResourceProvider, + hostResourceProvider, hostComponentResourceProvider, configGroupResourceProvider); + + RequestStatus requestStatus = provider.createResources(request); + + assertEquals(RequestStatus.Status.InProgress, requestStatus.getStatus()); + + Set<StackServiceRequest> stackServiceRequests = stackServiceRequestCapture.getValue(); + assertEquals(1, stackServiceRequests.size()); + StackServiceRequest ssr = stackServiceRequests.iterator().next(); + assertNull(ssr.getServiceName()); + assertEquals("test", ssr.getStackName()); + assertEquals("1.23", ssr.getStackVersion()); + + Set<StackServiceComponentRequest> stackServiceComponentRequests1 = serviceComponentRequestCapture1.getValue(); + Set<StackServiceComponentRequest> stackServiceComponentRequests2 = serviceComponentRequestCapture2.getValue(); + assertEquals(1, stackServiceComponentRequests1.size()); + assertEquals(1, stackServiceComponentRequests2.size()); + StackServiceComponentRequest scr1 = stackServiceComponentRequests1.iterator().next(); + StackServiceComponentRequest scr2 = stackServiceComponentRequests2.iterator().next(); + assertNull(scr1.getComponentName()); + assertNull(scr2.getComponentName()); + assertEquals("1.23", scr1.getStackVersion()); + assertEquals("1.23", scr2.getStackVersion()); + assertEquals("test", scr1.getStackName()); + assertEquals("test", scr2.getStackName()); + assertTrue(scr1.getServiceName().equals("service1") || scr1.getServiceName().equals("service2")); + assertTrue(scr2.getServiceName().equals("service1") || scr2.getServiceName().equals("service2") && + ! scr2.getServiceName().equals(scr1.getServiceName())); + + Set<StackConfigurationRequest> serviceConfigurationRequest1 = serviceConfigurationRequestCapture1.getValue(); + Set<StackConfigurationRequest> serviceConfigurationRequest2 = serviceConfigurationRequestCapture2.getValue(); + assertEquals(1, serviceConfigurationRequest1.size()); + assertEquals(1, serviceConfigurationRequest2.size()); + StackConfigurationRequest configReq1 = serviceConfigurationRequest1.iterator().next(); + StackConfigurationRequest configReq2 = serviceConfigurationRequest2.iterator().next(); + assertNull(configReq1.getPropertyName()); + assertNull(configReq2.getPropertyName()); + assertEquals("1.23", configReq1.getStackVersion()); + assertEquals("1.23", configReq2.getStackVersion()); + assertEquals("test", configReq1.getStackName()); + assertEquals("test", configReq2.getStackName()); + assertTrue(configReq1.getServiceName().equals("service1") || configReq1.getServiceName().equals("service2")); + assertTrue(configReq2.getServiceName().equals("service1") || configReq2.getServiceName().equals("service2") && + ! configReq2.getServiceName().equals(configReq1.getServiceName())); + + ClusterRequest clusterRequest = createClusterRequestCapture.getValue(); + assertEquals(clusterName, clusterRequest.getClusterName()); + assertEquals("test-1.23", clusterRequest.getStackVersion()); + + Set<ClusterRequest> updateClusterRequest1 = updateClusterRequestCapture.getValue(); + Set<ClusterRequest> updateClusterRequest2 = updateClusterRequestCapture2.getValue(); + Set<ClusterRequest> updateClusterRequest3 = updateClusterRequestCapture3.getValue(); + Set<ClusterRequest> updateClusterRequest4 = updateClusterRequestCapture4.getValue(); + Set<ClusterRequest> updateClusterRequest5 = updateClusterRequestCapture5.getValue(); + assertEquals(1, updateClusterRequest1.size()); + assertEquals(1, updateClusterRequest2.size()); + assertEquals(1, updateClusterRequest3.size()); + assertEquals(1, updateClusterRequest4.size()); + assertEquals(1, updateClusterRequest5.size()); + ClusterRequest ucr1 = updateClusterRequest1.iterator().next(); + ClusterRequest ucr2 = updateClusterRequest2.iterator().next(); + ClusterRequest ucr3 = updateClusterRequest3.iterator().next(); + ClusterRequest ucr4 = updateClusterRequest4.iterator().next(); + ClusterRequest ucr5 = updateClusterRequest5.iterator().next(); + assertEquals(clusterName, ucr1.getClusterName()); + assertEquals(clusterName, ucr2.getClusterName()); + assertEquals(clusterName, ucr3.getClusterName()); + assertEquals(clusterName, ucr4.getClusterName()); + assertEquals(clusterName, ucr5.getClusterName()); + ConfigurationRequest cr1 = ucr1.getDesiredConfig(); + ConfigurationRequest cr2 = ucr2.getDesiredConfig(); + ConfigurationRequest cr3 = ucr3.getDesiredConfig(); + ConfigurationRequest cr4 = ucr4.getDesiredConfig(); + ConfigurationRequest cr5 = ucr5.getDesiredConfig(); + assertEquals("1", cr1.getVersionTag()); + assertEquals("1", cr2.getVersionTag()); + assertEquals("1", cr3.getVersionTag()); + assertEquals("1", cr4.getVersionTag()); + assertEquals("1", cr5.getVersionTag()); + Map<String, ConfigurationRequest> mapConfigRequests = new HashMap<String, ConfigurationRequest>(); + mapConfigRequests.put(cr1.getType(), cr1); + mapConfigRequests.put(cr2.getType(), cr2); + mapConfigRequests.put(cr3.getType(), cr3); + mapConfigRequests.put(cr4.getType(), cr4); + mapConfigRequests.put(cr5.getType(), cr5); + assertEquals(5, mapConfigRequests.size()); + ConfigurationRequest globalConfigRequest = mapConfigRequests.get("global"); + assertEquals(4, globalConfigRequest.getProperties().size()); + assertEquals("hadoop", globalConfigRequest.getProperties().get("user_group")); + assertEquals("ambari-qa", globalConfigRequest.getProperties().get("smokeuser")); + assertEquals("default@REPLACEME.NOWHERE", globalConfigRequest.getProperties().get("nagios_contact")); + assertEquals("oozie", globalConfigRequest.getProperties().get("oozie_user")); + assertNotNull(globalConfigRequest.getPropertiesAttributes()); + assertEquals(0, globalConfigRequest.getPropertiesAttributes().size()); + ConfigurationRequest hiveEnvConfigRequest = mapConfigRequests.get("hive-env"); + assertEquals("New MySQL Database", hiveEnvConfigRequest.getProperties().get("hive_database")); + assertNotNull(hiveEnvConfigRequest.getPropertiesAttributes()); + assertEquals(1, hiveEnvConfigRequest.getPropertiesAttributes().size()); + Map<String, String> hiveEnvFinalAttrs = hiveEnvConfigRequest.getPropertiesAttributes().get("final"); + assertEquals(1, hiveEnvFinalAttrs.size()); + assertEquals("true", hiveEnvFinalAttrs.get("hive_database")); + ConfigurationRequest hdfsConfigRequest = mapConfigRequests.get("hdfs-site"); + assertEquals(1, hdfsConfigRequest.getProperties().size()); + assertEquals("value2", hdfsConfigRequest.getProperties().get("property2")); + Map<String, Map<String, String>> hdfsAttributes = hdfsConfigRequest.getPropertiesAttributes(); + assertTrue(hdfsAttributes.containsKey("final")); + assertEquals(1, hdfsAttributes.get("final").size()); + assertEquals("true", hdfsAttributes.get("final").get("property2")); + ConfigurationRequest coreConfigRequest = mapConfigRequests.get("core-site"); + assertEquals(5, coreConfigRequest.getProperties().size()); + assertEquals("value2", coreConfigRequest.getProperties().get("property1")); + assertEquals("value3", coreConfigRequest.getProperties().get("property3")); + assertEquals("*", coreConfigRequest.getProperties().get("hadoop.proxyuser.oozie.hosts")); + assertEquals("users", coreConfigRequest.getProperties().get("hadoop.proxyuser.oozie.groups")); + assertEquals("new.property.value", coreConfigRequest.getProperties().get("new.property")); + assertNotNull(coreConfigRequest.getPropertiesAttributes()); + assertEquals(1, coreConfigRequest.getPropertiesAttributes().size()); + Map<String, String> coreFinalAttrs = coreConfigRequest.getPropertiesAttributes().get("final"); + assertEquals(2, coreFinalAttrs.size()); + assertEquals("false", coreFinalAttrs.get("property1")); + assertEquals("true", coreFinalAttrs.get("new.property")); + ConfigurationRequest hiveConfigRequest = mapConfigRequests.get("hive-site"); + assertEquals(1, hiveConfigRequest.getProperties().size()); + assertEquals("host.domain:12345", hiveConfigRequest.getProperties().get("javax.jdo.option.ConnectionURL")); + assertNotNull(hiveConfigRequest.getPropertiesAttributes()); + assertEquals(0, hiveConfigRequest.getPropertiesAttributes().size()); + + assertNull(updateClusterPropertyMapCapture.getValue()); + assertNull(updateClusterPropertyMapCapture2.getValue()); + assertNull(updateClusterPropertyMapCapture3.getValue()); + assertNull(updateClusterPropertyMapCapture4.getValue()); + + Request serviceRequest = serviceRequestCapture.getValue(); + assertEquals(2, serviceRequest.getProperties().size()); + Request componentRequest = componentRequestCapture.getValue(); + Request componentRequest2 = componentRequestCapture2.getValue(); + assertEquals(3, componentRequest.getProperties().size()); + Set<String> componentRequest1Names = new HashSet<String>(); + for (Map<String, Object> componentRequest1Properties : componentRequest.getProperties()) { + assertEquals(3, componentRequest1Properties.size()); + assertEquals(clusterName, componentRequest1Properties.get("ServiceComponentInfo/cluster_name")); + assertEquals("service1", componentRequest1Properties.get("ServiceComponentInfo/service_name")); + componentRequest1Names.add((String) componentRequest1Properties.get("ServiceComponentInfo/component_name")); + } + assertTrue(componentRequest1Names.contains("component1") && componentRequest1Names.contains("component2") + && componentRequest1Names.contains("MYSQL_SERVER")); + assertEquals(1, componentRequest2.getProperties().size()); + Map<String, Object> componentRequest2Properties = componentRequest2.getProperties().iterator().next(); + assertEquals(clusterName, componentRequest2Properties.get("ServiceComponentInfo/cluster_name")); + assertEquals("service2", componentRequest2Properties.get("ServiceComponentInfo/service_name")); + assertEquals("component3", componentRequest2Properties.get("ServiceComponentInfo/component_name")); + Request hostRequest = hostRequestCapture.getValue(); + assertEquals(1, hostRequest.getProperties().size()); + assertEquals(clusterName, hostRequest.getProperties().iterator().next().get("Hosts/cluster_name")); + assertEquals("host.domain", hostRequest.getProperties().iterator().next().get("Hosts/host_name")); + Request hostComponentRequest = hostComponentRequestCapture.getValue(); + assertEquals(4, hostComponentRequest.getProperties().size()); + Set<String> componentNames = new HashSet<String>(); + for (Map<String, Object> hostComponentProperties : hostComponentRequest.getProperties()) { + assertEquals(3, hostComponentProperties.size()); + assertEquals(clusterName, hostComponentProperties.get("HostRoles/cluster_name")); + assertEquals("host.domain", hostComponentProperties.get("HostRoles/host_name")); + componentNames.add((String) hostComponentProperties.get("HostRoles/component_name")); + } + assertTrue(componentNames.contains("component1") && componentNames.contains("component2") && + componentNames.contains("component3") && componentNames.contains("MYSQL_SERVER")); + + Set<ConfigGroupRequest> configGroupRequests = configGroupRequestCapture.getValue(); + assertEquals(1, configGroupRequests.size()); + ConfigGroupRequest configGroupRequest = configGroupRequests.iterator().next(); + assertEquals(clusterName, configGroupRequest.getClusterName()); + assertEquals("group1", configGroupRequest.getGroupName()); + assertEquals("service1", configGroupRequest.getTag()); + assertEquals("Host Group Configuration", configGroupRequest.getDescription()); + Set<String> hosts = configGroupRequest.getHosts(); + assertEquals(1, hosts.size()); + assertEquals("host.domain", hosts.iterator().next()); + assertEquals(1, configGroupRequest.getConfigs().size()); + + verify(blueprintDAO, managementController, request, response, blueprint, stackServiceResponse1, stackServiceResponse2, + stackServiceComponentResponse1, stackServiceComponentResponse2, stackServiceComponentResponse3, + stackServiceComponentResponse4, stackConfigurationResponse1, stackConfigurationResponse2, + stackConfigurationResponse3, stackConfigurationResponse4, stackConfigurationResponse5, blueprintConfig, + blueprintConfig2, hostGroup, hostGroupComponent1, hostGroupComponent2, hostGroupComponent3, hostGroupComponent4, + hostGroupConfig, serviceResourceProvider, componentResourceProvider, hostResourceProvider, + hostComponentResourceProvider, configGroupResourceProvider, persistKeyValue, metaInfo); + } + @SuppressWarnings("unchecked") @Test public void testBlueprintPropertyUpdaters() throws Exception { http://git-wip-us.apache.org/repos/asf/ambari/blob/2885f191/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/StackConfigurationResourceProviderTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/StackConfigurationResourceProviderTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/StackConfigurationResourceProviderTest.java index 7a7b07f..d673671 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/StackConfigurationResourceProviderTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/StackConfigurationResourceProviderTest.java @@ -23,7 +23,9 @@ import static org.easymock.EasyMock.expect; import static org.easymock.EasyMock.replay; import static org.easymock.EasyMock.verify; +import java.util.HashMap; import java.util.HashSet; +import java.util.Map; import java.util.Set; import org.apache.ambari.server.controller.AmbariManagementController; @@ -41,18 +43,20 @@ public class StackConfigurationResourceProviderTest { private static final String PROPERTY_VALUE = "value"; private static final String PROPERTY_DESC = "Desc"; private static final String TYPE = "type.xml"; - private static final Boolean FINAL = false; @Test public void testGetResources() throws Exception{ + Map<String, String> attributes = new HashMap<String, String>(); + attributes.put("final", "true"); + Resource.Type type = Resource.Type.StackConfiguration; AmbariManagementController managementController = createMock(AmbariManagementController.class); Set<StackConfigurationResponse> allResponse = new HashSet<StackConfigurationResponse>(); - allResponse.add(new StackConfigurationResponse(PROPERTY_NAME, PROPERTY_VALUE, PROPERTY_DESC, TYPE, FINAL)); + allResponse.add(new StackConfigurationResponse(PROPERTY_NAME, PROPERTY_VALUE, PROPERTY_DESC, TYPE, attributes)); // set expectations expect(managementController.getStackConfigurations( @@ -93,14 +97,81 @@ public class StackConfigurationResourceProviderTest { resource.getPropertyValue(StackConfigurationResourceProvider.PROPERTY_DESCRIPTION_PROPERTY_ID); String propertyType = (String) resource.getPropertyValue(StackConfigurationResourceProvider.PROPERTY_TYPE_PROPERTY_ID); - Boolean propertyIsFinal = (Boolean) + String propertyIsFinal = (String) resource.getPropertyValue(StackConfigurationResourceProvider.PROPERTY_FINAL_PROPERTY_ID); Assert.assertEquals(PROPERTY_NAME, propertyName); Assert.assertEquals(PROPERTY_VALUE, propertyValue); Assert.assertEquals(PROPERTY_DESC, propertyDesc); Assert.assertEquals(TYPE, propertyType); - Assert.assertEquals(FINAL, propertyIsFinal); + Assert.assertEquals("true", propertyIsFinal); + + } + + // verify + verify(managementController); + } + + @Test + public void testGetResources_noFinal() throws Exception{ + + Map<String, String> attributes = new HashMap<String, String>(); + + Resource.Type type = Resource.Type.StackConfiguration; + + AmbariManagementController managementController = createMock(AmbariManagementController.class); + + Set<StackConfigurationResponse> allResponse = new HashSet<StackConfigurationResponse>(); + + allResponse.add(new StackConfigurationResponse(PROPERTY_NAME, PROPERTY_VALUE, PROPERTY_DESC, TYPE, attributes)); + + // set expectations + expect(managementController.getStackConfigurations( + AbstractResourceProviderTest.Matcher.getStackConfigurationRequestSet(null, null, null, null))). + andReturn(allResponse).times(1); + // replay + replay(managementController); + + ResourceProvider provider = AbstractControllerResourceProvider.getResourceProvider( + type, + PropertyHelper.getPropertyIds(type), + PropertyHelper.getKeyPropertyIds(type), + managementController); + + Set<String> propertyIds = new HashSet<String>(); + + propertyIds.add(StackConfigurationResourceProvider.STACK_NAME_PROPERTY_ID); + propertyIds.add(StackConfigurationResourceProvider.STACK_VERSION_PROPERTY_ID); + propertyIds.add(StackConfigurationResourceProvider.SERVICE_NAME_PROPERTY_ID); + propertyIds.add(StackConfigurationResourceProvider.PROPERTY_NAME_PROPERTY_ID); + propertyIds.add(StackConfigurationResourceProvider.PROPERTY_VALUE_PROPERTY_ID); + propertyIds.add(StackConfigurationResourceProvider.PROPERTY_DESCRIPTION_PROPERTY_ID); + propertyIds.add(StackConfigurationResourceProvider.PROPERTY_TYPE_PROPERTY_ID); + propertyIds.add(StackConfigurationResourceProvider.PROPERTY_FINAL_PROPERTY_ID); + + // create the request + Request request = PropertyHelper.getReadRequest(propertyIds); + + // get all ... no predicate + Set<Resource> resources = provider.getResources(request, null); + + Assert.assertEquals(allResponse.size(), resources.size()); + + for (Resource resource : resources) { + String propertyName = (String) resource.getPropertyValue(StackConfigurationResourceProvider.PROPERTY_NAME_PROPERTY_ID); + String propertyValue = (String) resource.getPropertyValue(StackConfigurationResourceProvider.PROPERTY_VALUE_PROPERTY_ID); + String propertyDesc = (String) + resource.getPropertyValue(StackConfigurationResourceProvider.PROPERTY_DESCRIPTION_PROPERTY_ID); + String propertyType = (String) + resource.getPropertyValue(StackConfigurationResourceProvider.PROPERTY_TYPE_PROPERTY_ID); + String propertyIsFinal = (String) + resource.getPropertyValue(StackConfigurationResourceProvider.PROPERTY_FINAL_PROPERTY_ID); + + Assert.assertEquals(PROPERTY_NAME, propertyName); + Assert.assertEquals(PROPERTY_VALUE, propertyValue); + Assert.assertEquals(PROPERTY_DESC, propertyDesc); + Assert.assertEquals(TYPE, propertyType); + Assert.assertEquals("false", propertyIsFinal); } http://git-wip-us.apache.org/repos/asf/ambari/blob/2885f191/ambari-server/src/test/java/org/apache/ambari/server/state/PropertyInfoTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/state/PropertyInfoTest.java b/ambari-server/src/test/java/org/apache/ambari/server/state/PropertyInfoTest.java new file mode 100644 index 0000000..c4eff8c --- /dev/null +++ b/ambari-server/src/test/java/org/apache/ambari/server/state/PropertyInfoTest.java @@ -0,0 +1,62 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ambari.server.state; + +import org.apache.ambari.server.api.util.StackExtensionHelper; +import org.apache.ambari.server.state.stack.ConfigurationXml; +import org.junit.Test; + +import java.io.File; +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.*; + +public class PropertyInfoTest { + + @Test + public void testGetAttributesMap() throws Exception { + Map<String, String> attributes; + File configFile = new File("./src/test/resources/stacks/HDP/2.0.8/services/HDFS/configuration/hdfs-site.xml"); + ConfigurationXml configuration = StackExtensionHelper.unmarshal(ConfigurationXml.class, configFile); + List<PropertyInfo> properties = configuration.getProperties(); + PropertyInfo dfsNameDir = properties.get(0); + assertNotNull(dfsNameDir); + assertEquals("dfs.name.dir", dfsNameDir.getName()); + attributes = dfsNameDir.getAttributesMap(); + assertEquals(1, attributes.size()); + assertTrue(attributes.containsKey("final")); + assertEquals("true", attributes.get("final")); + + PropertyInfo dfsSupportAppend = properties.get(1); + assertNotNull(dfsSupportAppend); + assertEquals("dfs.support.append", dfsSupportAppend.getName()); + attributes = dfsSupportAppend.getAttributesMap(); + assertEquals(2, attributes.size()); + assertTrue(attributes.containsKey("final")); + assertEquals("true", attributes.get("final")); + assertTrue(attributes.containsKey("deletable")); + assertEquals("false", attributes.get("deletable")); + + PropertyInfo dfsWebhdfsEnabled = properties.get(2); + assertNotNull(dfsWebhdfsEnabled); + assertEquals("dfs.webhdfs.enabled", dfsWebhdfsEnabled.getName()); + attributes = dfsWebhdfsEnabled.getAttributesMap(); + assertEquals(0, attributes.size()); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/2885f191/ambari-server/src/test/resources/stacks/HDP/2.0.8/services/HDFS/configuration/hdfs-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/resources/stacks/HDP/2.0.8/services/HDFS/configuration/hdfs-site.xml b/ambari-server/src/test/resources/stacks/HDP/2.0.8/services/HDFS/configuration/hdfs-site.xml new file mode 100644 index 0000000..c40fbd0 --- /dev/null +++ b/ambari-server/src/test/resources/stacks/HDP/2.0.8/services/HDFS/configuration/hdfs-site.xml @@ -0,0 +1,45 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> + +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<configuration> + + <property> + <name>dfs.name.dir</name> + <value></value> + <description>Determines where on the local filesystem the DFS name node + should store the name table.</description> + <final>true</final> + </property> + + <property> + <name>dfs.support.append</name> + <value>true</value> + <description>to enable dfs append</description> + <final>true</final> + <deletable>false</deletable> + </property> + + <property> + <name>dfs.webhdfs.enabled</name> + <value>true</value> + <description>to enable webhdfs</description> + </property> + +</configuration>
