AMBARI-18355: Introduce conditional dependencies in stack defition to handle blueprint validation gracefully (Amruta Borkar via dili)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/f6124a05 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/f6124a05 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/f6124a05 Branch: refs/heads/branch-dev-patch-upgrade Commit: f6124a056d2a8ed16bec917775b9d3554ab5d74d Parents: c153cfb Author: Di Li <d...@apache.org> Authored: Fri Oct 7 13:34:54 2016 -0400 Committer: Di Li <d...@apache.org> Committed: Fri Oct 7 13:34:54 2016 -0400 ---------------------------------------------------------------------- .../server/state/DependencyConditionInfo.java | 102 +++++++++++++++++++ .../ambari/server/state/DependencyInfo.java | 36 ++++++- .../server/topology/BlueprintValidatorImpl.java | 13 +++ .../common-services/HDFS/2.1.0.2.0/metainfo.xml | 44 ++++++++ .../topology/BlueprintValidatorImplTest.java | 75 +++++++++++++- 5 files changed, 268 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/f6124a05/ambari-server/src/main/java/org/apache/ambari/server/state/DependencyConditionInfo.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/DependencyConditionInfo.java b/ambari-server/src/main/java/org/apache/ambari/server/state/DependencyConditionInfo.java new file mode 100644 index 0000000..84e186f --- /dev/null +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/DependencyConditionInfo.java @@ -0,0 +1,102 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ambari.server.state; + +import java.util.Map; +import java.util.Objects; +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.adapters.XmlAdapter; +import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter; + +import org.apache.commons.lang.NotImplementedException; + +/** + * Represents stack component dependency condition information. + */ +@XmlJavaTypeAdapter(DependencyConditionAdapter.class) +public abstract class DependencyConditionInfo { + public abstract boolean isResolved(Map<String, Map<String, String>> properties); +} + +class PropertyExistsDependencyCondition extends DependencyConditionInfo{ + + protected final String configType; + protected final String property; + public PropertyExistsDependencyCondition( String configType, String property) { + this.configType = Objects.requireNonNull(configType, "Config Type must not be null."); + this.property = Objects.requireNonNull(property, "Property Name must not be null."); + } + + @Override + public boolean isResolved(Map<String, Map<String, String>> properties) { + return (properties.get(configType).containsKey(property)); + } +} + +class PropertyValueEqualsDependencyCondition extends PropertyExistsDependencyCondition { + + protected final String propertyValue; + public PropertyValueEqualsDependencyCondition(String configType, String property, String propertyValue) { + super(configType, property); + this.propertyValue = Objects.requireNonNull(propertyValue, "Property value must not be null."); + } + + @Override + public boolean isResolved(Map<String, Map<String, String>> properties) { + return (super.isResolved(properties) && propertyValue.equals(properties.get(configType).get(property))); + } +} + +class DependencyConditionAdapter extends XmlAdapter<DependencyConditionAdapter.AdaptedDependencyCondition, DependencyConditionInfo> { + + static class AdaptedDependencyCondition{ + @XmlElement + private String configType; + @XmlElement + private String property; + @XmlElement + private String propertyValue; + @XmlElement(name="condition-type") + private String conditionType; + } + + @Override + public AdaptedDependencyCondition marshal(DependencyConditionInfo arg0) throws Exception { + throw new NotImplementedException(); + } + + @Override + public DependencyConditionInfo unmarshal(AdaptedDependencyCondition adaptedDependencyCondition) throws Exception { + if (null == adaptedDependencyCondition) { + return null; + } + DependencyConditionInfo dependencyConditionInfo = null; + switch (adaptedDependencyCondition.conditionType) { + case "IF-PROPERTY-EXISTS": + dependencyConditionInfo = new PropertyExistsDependencyCondition(adaptedDependencyCondition.configType, adaptedDependencyCondition.property); + break; + case "PROPERTY-VALUE-EQUALS": + dependencyConditionInfo = new PropertyValueEqualsDependencyCondition(adaptedDependencyCondition.configType, adaptedDependencyCondition.property, adaptedDependencyCondition.propertyValue); + break; + default: + throw new IllegalArgumentException("Specified condition type is not not supported " + adaptedDependencyCondition.conditionType); + } + return dependencyConditionInfo; + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/f6124a05/ambari-server/src/main/java/org/apache/ambari/server/state/DependencyInfo.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/DependencyInfo.java b/ambari-server/src/main/java/org/apache/ambari/server/state/DependencyInfo.java index e3db662..3c88401 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/DependencyInfo.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/DependencyInfo.java @@ -20,6 +20,10 @@ package org.apache.ambari.server.state; import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlElementWrapper; +import javax.xml.bind.annotation.XmlElements; +import java.util.ArrayList; +import java.util.List; /** * Represents stack component dependency information. @@ -54,7 +58,10 @@ public class DependencyInfo { @XmlElement(name="auto-deploy") private AutoDeployInfo m_autoDeploy; - + /** + * Conditions for Component dependency to other components. + */ + private List<DependencyConditionInfo> dependencyConditions = new ArrayList<DependencyConditionInfo>(); /** * Setter for name property. * @@ -135,6 +142,33 @@ public class DependencyInfo { public String getServiceName() { return serviceName; } + /** + * Get the dependencyConditions list + * + * @return dependencyConditions + */ + @XmlElementWrapper(name="conditions") + @XmlElements(@XmlElement(name="condition")) + public List<DependencyConditionInfo> getDependencyConditions() { + return dependencyConditions; + } + + /** + * Set dependencyConditions + * + * @param dependencyConditions + */ + public void setDependencyConditions(List<DependencyConditionInfo> dependencyConditions) { + this.dependencyConditions = dependencyConditions; + } + + /** + * Confirms if dependency have any condition or not + * @return true if dependencies are based on a condition + */ + public boolean hasDependencyConditions(){ + return !(dependencyConditions == null || dependencyConditions.isEmpty()); + } @Override public String toString() { http://git-wip-us.apache.org/repos/asf/ambari/blob/f6124a05/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintValidatorImpl.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintValidatorImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintValidatorImpl.java index a5f33ff..7bbe0db 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintValidatorImpl.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintValidatorImpl.java @@ -30,6 +30,7 @@ import java.util.regex.Pattern; import org.apache.ambari.server.controller.internal.Stack; import org.apache.ambari.server.state.AutoDeployInfo; +import org.apache.ambari.server.state.DependencyConditionInfo; import org.apache.ambari.server.state.DependencyInfo; import org.apache.ambari.server.utils.SecretReference; import org.apache.ambari.server.utils.VersionUtils; @@ -289,6 +290,18 @@ public class BlueprintValidatorImpl implements BlueprintValidator { AutoDeployInfo autoDeployInfo = dependency.getAutoDeploy(); boolean resolved = false; + //check if conditions are met, if any + if(dependency.hasDependencyConditions()) { + boolean conditionsSatisfied = true; + for (DependencyConditionInfo dependencyCondition : dependency.getDependencyConditions()) { + if (!dependencyCondition.isResolved(blueprint.getConfiguration().getFullProperties())) { + conditionsSatisfied = false; + } + } + if(!conditionsSatisfied){ + continue; + } + } if (dependencyScope.equals("cluster")) { Collection<String> missingDependencyInfo = verifyComponentCardinalityCount( componentName, new Cardinality("1+"), autoDeployInfo); http://git-wip-us.apache.org/repos/asf/ambari/blob/f6124a05/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/metainfo.xml b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/metainfo.xml index 65d166a..dfbbd55 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/metainfo.xml +++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/metainfo.xml @@ -32,6 +32,50 @@ <cardinality>1-2</cardinality> <versionAdvertised>true</versionAdvertised> <reassignAllowed>true</reassignAllowed> + <dependencies> + <dependency> + <name>HDFS/ZKFC</name> + <scope>host</scope> + <auto-deploy> + <enabled>false</enabled> + </auto-deploy> + <conditions> + <condition> + <condition-type>IF-PROPERTY-EXISTS</condition-type> + <configType>hdfs-site</configType> + <property>dfs.nameservices</property> + </condition> + </conditions> + </dependency> + <dependency> + <name>ZOOKEEPER/ZOOKEEPER_SERVER</name> + <scope>host</scope> + <auto-deploy> + <enabled>false</enabled> + </auto-deploy> + <conditions> + <condition> + <condition-type>IF-PROPERTY-EXISTS</condition-type> + <configType>hdfs-site</configType> + <property>dfs.nameservices</property> + </condition> + </conditions> + </dependency> + <dependency> + <name>HDFS/JOURNALNODE</name> + <scope>host</scope> + <auto-deploy> + <enabled>false</enabled> + </auto-deploy> + <conditions> + <condition> + <condition-type>IF-PROPERTY-EXISTS</condition-type> + <configType>hdfs-site</configType> + <property>dfs.nameservices</property> + </condition> + </conditions> + </dependency> + </dependencies> <commandScript> <script>scripts/namenode.py</script> <scriptType>PYTHON</scriptType> http://git-wip-us.apache.org/repos/asf/ambari/blob/f6124a05/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintValidatorImplTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintValidatorImplTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintValidatorImplTest.java index b1de8ef..1501c53 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintValidatorImplTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintValidatorImplTest.java @@ -34,7 +34,9 @@ import java.util.Map; import org.apache.ambari.server.controller.internal.Stack; import org.apache.ambari.server.state.AutoDeployInfo; import org.apache.ambari.server.state.ComponentInfo; +import org.apache.ambari.server.state.DependencyConditionInfo; import org.apache.ambari.server.state.DependencyInfo; +import org.easymock.EasyMock; import org.easymock.EasyMockRule; import org.easymock.Mock; import org.easymock.MockType; @@ -66,15 +68,22 @@ public class BlueprintValidatorImplTest { @Mock(type = MockType.NICE) private DependencyInfo dependency1; + @Mock(type = MockType.NICE) + private DependencyInfo dependency2; @Mock(type = MockType.NICE) private ComponentInfo dependencyComponentInfo; + @Mock(type = MockType.NICE) + private DependencyConditionInfo dependencyConditionInfo1; + @Mock(type = MockType.NICE) + private DependencyConditionInfo dependencyConditionInfo2; private final Collection<String> group1Components = new ArrayList<String>(); private final Collection<String> group2Components = new ArrayList<String>(); private final Collection<String> services = new ArrayList<String>(); private Collection<DependencyInfo> dependencies1 = new ArrayList<DependencyInfo>(); + private List<DependencyConditionInfo> dependenciesConditionInfos1 = new ArrayList<DependencyConditionInfo>(); private AutoDeployInfo autoDeploy = new AutoDeployInfo(); private Map<String, Map<String, String>> configProperties = new HashMap<String, Map<String, String>>(); private Configuration configuration = new Configuration(configProperties, Collections.<String, Map<String, Map<String, String>>>emptyMap()); @@ -105,13 +114,15 @@ public class BlueprintValidatorImplTest { expect(stack.getCardinality("component1")).andReturn(new Cardinality("1")); expect(stack.getCardinality("component2")).andReturn(new Cardinality("1+")); expect(stack.getCardinality("component3")).andReturn(new Cardinality("1+")); + dependenciesConditionInfos1.add(dependencyConditionInfo1); + dependenciesConditionInfos1.add(dependencyConditionInfo2); expect(blueprint.getConfiguration()).andReturn(configuration).anyTimes(); } @After public void tearDown() { - reset(blueprint, stack, group1, group2, dependency1); + reset(blueprint, stack, group1, group2, dependency1, dependency2, dependencyConditionInfo1, dependencyConditionInfo2); } @Test @@ -341,4 +352,66 @@ public class BlueprintValidatorImplTest { verify(group1); } + @Test(expected=InvalidTopologyException.class) + public void testWhenComponentIsConditionallyDependentAndOnlyOneOfTheConditionsIsSatisfied() throws Exception { + // GIVEN + hostGroups.clear(); + hostGroups.put("group1", group1); + + group1Components.add("component-1"); + dependencies1.add(dependency1); + dependencies1.add(dependency2); + services.addAll(Collections.singleton("service-1")); + + + expect(blueprint.getHostGroupsForComponent("component-1")).andReturn(Arrays.asList(group1)).anyTimes(); + expect(blueprint.getName()).andReturn("blueprint-1").anyTimes(); + Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>(); + Map<String, String> typeProps = new HashMap<String, String>(); + typeProps.put("yarn.resourcemanager.hostname", "testhost"); + properties.put("yarn-site", typeProps); + + Configuration clusterConfig = new Configuration(properties, + Collections.<String, Map<String, Map<String, String>>>emptyMap()); + + Cardinality cardinality = new Cardinality("1"); + + expect(stack.getComponents("service-1")).andReturn(Arrays.asList("component-1")).anyTimes(); + expect(stack.getAutoDeployInfo("component-1")).andReturn(autoDeploy).anyTimes(); + expect(stack.getDependenciesForComponent("component-1")).andReturn(dependencies1).anyTimes(); + expect(stack.getCardinality("component-1")).andReturn(cardinality).anyTimes(); + + AutoDeployInfo dependencyAutoDeploy = null; + + expect(dependency1.getScope()).andReturn("host").anyTimes(); + expect(dependency1.getAutoDeploy()).andReturn(dependencyAutoDeploy).anyTimes(); + expect(dependency1.getComponentName()).andReturn("component-d").anyTimes(); + expect(dependency1.getServiceName()).andReturn("service-d").anyTimes(); + expect(dependency1.getName()).andReturn("dependency-1").anyTimes(); + expect(dependency1.hasDependencyConditions()).andReturn(true).anyTimes(); + expect(dependency1.getDependencyConditions()).andReturn(dependenciesConditionInfos1).anyTimes(); + expect(dependency2.getScope()).andReturn("host").anyTimes(); + expect(dependency2.getAutoDeploy()).andReturn(dependencyAutoDeploy).anyTimes(); + expect(dependency2.getComponentName()).andReturn("component-d").anyTimes(); + expect(dependency2.getServiceName()).andReturn("service-d").anyTimes(); + expect(dependency2.getName()).andReturn("dependency-2").anyTimes(); + expect(dependency2.hasDependencyConditions()).andReturn(false).anyTimes(); + + expect(dependencyConditionInfo1.isResolved(EasyMock.anyObject(Map.class))).andReturn(true).anyTimes(); + expect(dependencyConditionInfo2.isResolved(EasyMock.anyObject(Map.class))).andReturn(false).anyTimes(); + + + expect(dependencyComponentInfo.isClient()).andReturn(false).anyTimes(); + expect(stack.getComponentInfo("component-d")).andReturn(dependencyComponentInfo).anyTimes(); + + replay(blueprint, stack, group1, group2, dependency1, dependency2, dependencyComponentInfo,dependencyConditionInfo1,dependencyConditionInfo2); + + // WHEN + BlueprintValidator validator = new BlueprintValidatorImpl(blueprint); + validator.validateTopology(); + + // THEN + verify(group1); + + } }