Repository: ambari Updated Branches: refs/heads/trunk c1156d22b -> f674a5678
AMBARI-17146 Implement config values trimming for deployment via blueprint (dsen) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/f674a567 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/f674a567 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/f674a567 Branch: refs/heads/trunk Commit: f674a56783708d09704863b3f18507037b1f02d2 Parents: c1156d2 Author: Dmytro Sen <[email protected]> Authored: Thu Jun 16 17:40:11 2016 +0300 Committer: Dmytro Sen <[email protected]> Committed: Thu Jun 16 17:40:11 2016 +0300 ---------------------------------------------------------------------- .../BlueprintConfigurationProcessor.java | 31 ++++++++ .../internal/DefaultTrimmingStrategy.java | 31 ++++++++ .../DeleteSpacesAtTheEndTrimmingStrategy.java | 34 +++++++++ .../internal/DirectoriesTrimmingStrategy.java | 32 ++++++++ .../internal/PasswordTrimmingStrategy.java | 31 ++++++++ .../PropertyValueTrimmingStrategyDefiner.java | 80 ++++++++++++++++++++ .../server/controller/internal/Stack.java | 7 ++ .../controller/internal/TrimmingStrategy.java | 24 ++++++ .../BlueprintConfigurationProcessorTest.java | 60 ++++++++++++++- .../server/controller/internal/StackTest.java | 1 + 10 files changed, 330 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/f674a567/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java index 9094698..1e56bef 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java @@ -334,9 +334,40 @@ public class BlueprintConfigurationProcessor { setMissingConfigurations(clusterConfig, configTypesUpdated); + trimProperties(clusterConfig, clusterTopology); + return configTypesUpdated; } + private void trimProperties(Configuration clusterConfig, ClusterTopology clusterTopology) { + Blueprint blueprint = clusterTopology.getBlueprint(); + Stack stack = blueprint.getStack(); + + Map<String, Map<String, String>> configTypes = clusterConfig.getFullProperties(); + for (String configType : configTypes.keySet()) { + Map<String,String> properties = configTypes.get(configType); + for (String propertyName : properties.keySet()) { + trimPropertyValue(clusterConfig, stack, configType, properties, propertyName); + } + } + } + + private void trimPropertyValue(Configuration clusterConfig, Stack stack, String configType, Map<String, String> properties, String propertyName) { + if (propertyName != null && properties.get(propertyName) != null) { + + TrimmingStrategy trimmingStrategy = + PropertyValueTrimmingStrategyDefiner.defineTrimmingStrategy(stack, propertyName, configType); + String oldValue = properties.get(propertyName); + String newValue = trimmingStrategy.trim(oldValue); + + if (!newValue.equals(oldValue)){ + LOG.debug(String.format("Changing value for config %s property %s from [%s] to [%s]", + configType, propertyName, oldValue, newValue)); + clusterConfig.setProperty(configType, propertyName, newValue); + } + } + } + /** * Returns true if property should be retained with default value instead of deleting * TODO: This is a temporary work-around till BP integrates with stack advisor http://git-wip-us.apache.org/repos/asf/ambari/blob/f674a567/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/DefaultTrimmingStrategy.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/DefaultTrimmingStrategy.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/DefaultTrimmingStrategy.java new file mode 100644 index 0000000..8810bcb --- /dev/null +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/DefaultTrimmingStrategy.java @@ -0,0 +1,31 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ambari.server.controller.internal; + +public class DefaultTrimmingStrategy implements TrimmingStrategy { + @Override + public String trim(String stringToTrim) { + return stringToTrim.trim(); + } + + @Override + public String toString(){ + return "DefaultTrimmingStrategy"; + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/f674a567/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/DeleteSpacesAtTheEndTrimmingStrategy.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/DeleteSpacesAtTheEndTrimmingStrategy.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/DeleteSpacesAtTheEndTrimmingStrategy.java new file mode 100644 index 0000000..19ba1ec --- /dev/null +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/DeleteSpacesAtTheEndTrimmingStrategy.java @@ -0,0 +1,34 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ambari.server.controller.internal; + +public class DeleteSpacesAtTheEndTrimmingStrategy implements TrimmingStrategy { + @Override + public String trim(String stringToTrim) { + if (" ".equals(stringToTrim)) { + return stringToTrim; + } + return stringToTrim.replaceAll("\\s+$", ""); + } + + @Override + public String toString(){ + return "DeleteSpacesAtTheEndTrimmingStrategy"; + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/f674a567/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/DirectoriesTrimmingStrategy.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/DirectoriesTrimmingStrategy.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/DirectoriesTrimmingStrategy.java new file mode 100644 index 0000000..47231c6 --- /dev/null +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/DirectoriesTrimmingStrategy.java @@ -0,0 +1,32 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ambari.server.controller.internal; + +public class DirectoriesTrimmingStrategy implements TrimmingStrategy { + + @Override + public String trim(String stringToTrim) { + return stringToTrim.replaceAll("\\s*,+\\s*", ",").trim(); + } + + @Override + public String toString(){ + return "DirectoriesTrimmingStrategy"; + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/f674a567/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/PasswordTrimmingStrategy.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/PasswordTrimmingStrategy.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/PasswordTrimmingStrategy.java new file mode 100644 index 0000000..1407e51 --- /dev/null +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/PasswordTrimmingStrategy.java @@ -0,0 +1,31 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ambari.server.controller.internal; + +public class PasswordTrimmingStrategy implements TrimmingStrategy { + @Override + public String trim(String stringToTrim) { + return stringToTrim; + } + + @Override + public String toString(){ + return "PasswordTrimmingStrategy"; + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/f674a567/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/PropertyValueTrimmingStrategyDefiner.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/PropertyValueTrimmingStrategyDefiner.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/PropertyValueTrimmingStrategyDefiner.java new file mode 100644 index 0000000..877e27e --- /dev/null +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/PropertyValueTrimmingStrategyDefiner.java @@ -0,0 +1,80 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ambari.server.controller.internal; + + +import org.apache.ambari.server.state.ValueAttributesInfo; + +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +public class PropertyValueTrimmingStrategyDefiner { + static final Set<String> setOfUrlProperties = new HashSet<>(); + + static { + setOfUrlProperties.add("javax.jdo.option.ConnectionURL"); + setOfUrlProperties.add("oozie.service.JPAService.jdbc.url"); + } + + private static TrimmingStrategy getTrimmingStrategyForConfigProperty(Stack.ConfigProperty configProperty) { + if (configProperty != null) { + ValueAttributesInfo valueAttributesInfo = configProperty.getPropertyValueAttributes(); + if (valueAttributesInfo != null) { + String type = valueAttributesInfo.getType(); + if ("directory".equals(type) || "directories".equals(type)) { + return new DirectoriesTrimmingStrategy(); + } else if ("host".equals(type)) { + return new DefaultTrimmingStrategy(); + } + } + if (configProperty.getPropertyTypes() != null && configProperty.getPropertyTypes(). + contains(org.apache.ambari.server.state.PropertyInfo.PropertyType.PASSWORD)) { + return new PasswordTrimmingStrategy(); + } + } + return null; + } + + private static TrimmingStrategy getTrimmingStrategyByPropertyName(String propertyName) { + if (setOfUrlProperties.contains(propertyName)) { + return new DefaultTrimmingStrategy(); + } else { + return new DeleteSpacesAtTheEndTrimmingStrategy(); + } + } + + public static TrimmingStrategy defineTrimmingStrategy(Stack stack, String propertyName, String configType) { + TrimmingStrategy result = null; + String service = stack.getServiceForConfigType(configType); + if (service != null) { + Map<String, Stack.ConfigProperty> map = stack.getConfigurationPropertiesWithMetadata(service, configType); + if (map != null) { + Stack.ConfigProperty configProperty = map.get(propertyName); + if (configProperty != null) { + result = getTrimmingStrategyForConfigProperty(configProperty); + } + } + } + if (result == null) { + result = getTrimmingStrategyByPropertyName(propertyName); + } + return result; + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/f674a567/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/Stack.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/Stack.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/Stack.java index ad8d4f9..16f75ee 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/Stack.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/Stack.java @@ -40,6 +40,7 @@ import org.apache.ambari.server.state.ComponentInfo; import org.apache.ambari.server.state.DependencyInfo; import org.apache.ambari.server.state.PropertyDependencyInfo; import org.apache.ambari.server.state.PropertyInfo; +import org.apache.ambari.server.state.ValueAttributesInfo; import org.apache.ambari.server.topology.Cardinality; import org.apache.ambari.server.topology.Configuration; @@ -740,6 +741,7 @@ public class Stack { * Contains a configuration property's value and attributes. */ public static class ConfigProperty { + private ValueAttributesInfo propertyValueAttributes = null; private String name; private String value; private Map<String, String> attributes; @@ -755,6 +757,7 @@ public class Stack { this.propertyTypes = config.getPropertyType(); this.type = normalizeType(config.getType()); this.dependsOnProperties = config.getDependsOnProperties(); + this.propertyValueAttributes = config.getPropertyValueAttributes(); } public ConfigProperty(String type, String name, String value) { @@ -806,5 +809,9 @@ public class Stack { } return type; } + + public ValueAttributesInfo getPropertyValueAttributes() { + return propertyValueAttributes; + } } } http://git-wip-us.apache.org/repos/asf/ambari/blob/f674a567/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/TrimmingStrategy.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/TrimmingStrategy.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/TrimmingStrategy.java new file mode 100644 index 0000000..9928efc --- /dev/null +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/TrimmingStrategy.java @@ -0,0 +1,24 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ambari.server.controller.internal; + +public interface TrimmingStrategy { + String trim(String stringToTrim); +} + http://git-wip-us.apache.org/repos/asf/ambari/blob/f674a567/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java index cda8fb8..a927548 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java @@ -30,6 +30,8 @@ import java.util.Map; import java.util.Set; import com.google.common.collect.*; +import org.apache.ambari.server.controller.StackConfigurationResponse; +import org.apache.ambari.server.state.PropertyInfo; import org.apache.ambari.server.state.PropertyDependencyInfo; import org.apache.ambari.server.state.ServiceInfo; import org.apache.ambari.server.state.ValueAttributesInfo; @@ -4486,7 +4488,7 @@ public class BlueprintConfigurationProcessorTest { // customized stack calls for this test only // simulate the case of the stack object throwing a RuntimeException, to indicate a config error - expect(stack.getServiceForConfigType("hive-site")).andThrow(new RuntimeException("Expected Test Error")).atLeastOnce(); + expect(stack.getServiceForConfigType("hive-site")).andThrow(new RuntimeException("Expected Test Error")).once(); expect(stack.getConfigurationPropertiesWithMetadata("HIVE", "hive-site")).andReturn(mapOfMetadata).atLeastOnce(); Configuration clusterConfig = new Configuration(properties, Collections.<String, Map<String, Map<String, String>>>emptyMap()); @@ -7891,6 +7893,62 @@ public class BlueprintConfigurationProcessorTest { return advMap; } + @Test + public void testValuesTrimming() throws Exception { + reset(stack); + Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>(); + + Map<String, String> hdfsSite = new HashMap<String, String>(); + //default + hdfsSite.put("test.spaces", " spaces at the end should be deleted "); + hdfsSite.put("test.directories", " /all/spaces , should/be , deleted "); + hdfsSite.put("test.password", " stays, same "); + hdfsSite.put("test.single.space", " "); + hdfsSite.put("test.host", " https://just.trims "); + properties.put("hdfs-site", hdfsSite); + Map<String, Stack.ConfigProperty> propertyConfigs = new HashMap<>(); + + ValueAttributesInfo valueAttributesInfoDirs = new ValueAttributesInfo(); + valueAttributesInfoDirs.setType("directories"); + ValueAttributesInfo valueAttributesInfoHost = new ValueAttributesInfo(); + valueAttributesInfoHost.setType("host"); + + propertyConfigs.put("test.directories", new Stack.ConfigProperty( + new StackConfigurationResponse(null,null,null,null,"hdfs-site",null,null,null,valueAttributesInfoDirs,null))); + propertyConfigs.put("test.password", new Stack.ConfigProperty( + new StackConfigurationResponse(null,null,null,null,"hdfs-site",null,Collections.singleton(PropertyInfo.PropertyType.PASSWORD),null,null,null))); + propertyConfigs.put("test.host", new Stack.ConfigProperty( + new StackConfigurationResponse(null,null,null,null,"hdfs-site",null,null,null,valueAttributesInfoHost,null))); + expect(stack.getServiceForConfigType("hdfs-site")).andReturn("HDFS").anyTimes(); + expect(stack.getConfigurationPropertiesWithMetadata("HDFS", "hdfs-site")).andReturn(propertyConfigs).anyTimes(); + + Map<String, Map<String, String>> parentProperties = new HashMap<String, Map<String, String>>(); + Configuration parentClusterConfig = new Configuration(parentProperties, + Collections.<String, Map<String, Map<String, String>>>emptyMap()); + Configuration clusterConfig = new Configuration(properties, + Collections.<String, Map<String, Map<String, String>>>emptyMap(), parentClusterConfig); + + Collection<String> hgComponents1 = new HashSet<String>(); + TestHostGroup group1 = new TestHostGroup("group1", hgComponents1, Collections.singleton("host1")); + + Collection<TestHostGroup> hostGroups = Collections.singletonList(group1); + + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); + + configProcessor.doUpdateForClusterCreate(); + + assertEquals(" spaces at the end should be deleted", + clusterConfig.getPropertyValue("hdfs-site", "test.spaces")); + assertEquals("/all/spaces,should/be,deleted", + clusterConfig.getPropertyValue("hdfs-site", "test.directories")); + assertEquals( " stays, same ", + clusterConfig.getPropertyValue("hdfs-site", "test.password")); + assertEquals(" https://just.trims ".trim(), + clusterConfig.getPropertyValue("hdfs-site", "test.host")); + assertEquals(" ", + clusterConfig.getPropertyValue("hdfs-site", "test.single.space")); + } private static String createExportedAddress(String expectedPortNum, String expectedHostGroupName) { return createExportedHostName(expectedHostGroupName, expectedPortNum); http://git-wip-us.apache.org/repos/asf/ambari/blob/f674a567/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/StackTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/StackTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/StackTest.java index e70af3e..a6829e8 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/StackTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/StackTest.java @@ -147,6 +147,7 @@ public class StackTest { expect(mockResponse.getPropertyType()).andReturn(Collections.<PropertyInfo.PropertyType>emptySet()); expect(mockResponse.getType()).andReturn("test-type-one"); expect(mockResponse.getDependsOnProperties()).andReturn(setOfDependencyInfo); + expect(mockResponse.getPropertyValueAttributes()).andReturn(new ValueAttributesInfo()); mockSupport.replayAll();
