Repository: ambari Updated Branches: refs/heads/trunk f5594591e -> 8d3da8ef2
AMBARI-10176. Storm service check failed after disabling security (rlevas) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/8d3da8ef Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/8d3da8ef Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/8d3da8ef Branch: refs/heads/trunk Commit: 8d3da8ef2458cc49d9093bda605f2099e5683573 Parents: f559459 Author: Robert Levas <rle...@hortonworks.com> Authored: Wed Mar 25 09:13:45 2015 -0400 Committer: Robert Levas <rle...@hortonworks.com> Committed: Wed Mar 25 09:14:02 2015 -0400 ---------------------------------------------------------------------- .../server/controller/KerberosHelper.java | 49 ++++++++++++++-- .../kerberos/KerberosConfigDataFile.java | 4 ++ .../kerberos/KerberosConfigDataFileBuilder.java | 13 +++-- .../UpdateKerberosConfigsServerAction.java | 60 ++++++++++++++------ .../ambari/server/state/ConfigHelper.java | 11 +++- .../0.9.1.2.1/configuration/storm-site.xml | 5 -- .../kerberos/KerberosConfigDataFileTest.java | 26 +++++++-- .../UpdateKerberosConfigsServerActionTest.java | 3 +- 8 files changed, 130 insertions(+), 41 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/8d3da8ef/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelper.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelper.java index 01f7846..75062a1 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelper.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelper.java @@ -411,7 +411,7 @@ public class KerberosHelper { setAuthToLocalRules(kerberosDescriptor, cluster, kerberosDetails.getDefaultRealm(), configurations, kerberosConfigurations); for (Map.Entry<String, Map<String, String>> entry : kerberosConfigurations.entrySet()) { - configHelper.updateConfigType(cluster, ambariManagementController, entry.getKey(), entry.getValue(), + configHelper.updateConfigType(cluster, ambariManagementController, entry.getKey(), entry.getValue(), null, ambariManagementController.getAuthName(), String.format("Enabling Kerberos for %s", serviceName)); } } @@ -2253,7 +2253,8 @@ public class KerberosHelper { for (Map.Entry<String, String> configTypeEntry : properties.entrySet()) { kerberosConfDataFileBuilder.addRecord(type, configTypeEntry.getKey(), - configTypeEntry.getValue()); + configTypeEntry.getValue(), + KerberosConfigDataFile.OPERATION_TYPE_SET); } } } @@ -2373,9 +2374,21 @@ public class KerberosHelper { // If there are configurations to set, create a (temporary) data file to store the configuration // updates and fill it will the relevant configurations. if (!kerberosConfigurations.isEmpty()) { + Map<String, Collection<String>> configurationsToRemove = new HashMap<String, Collection<String>>(); File configFile = new File(dataDirectory, KerberosConfigDataFile.DATA_FILE_NAME); KerberosConfigDataFileBuilder kerberosConfDataFileBuilder = null; + // Fill the configurationsToRemove map with all Kerberos-related configurations. Values + // needed to be kept will have new values from the stack definition and thus pruned from + // this map. + for (Map.Entry<String, Map<String, String>> entry : kerberosConfigurations.entrySet()) { + configurationsToRemove.put(entry.getKey(), new HashSet<String>(entry.getValue().keySet())); + } + + // Remove cluster-env from the set of configurations to remove since it has no default set + // or properties and the logic below will remove all from this set - which is not desirable. + configurationsToRemove.remove("cluster-env"); + if (serviceComponentHosts != null) { Set<String> visitedServices = new HashSet<String>(); @@ -2395,10 +2408,18 @@ public class KerberosHelper { String filename = propertyInfo.getFilename(); if (filename != null) { - Map<String, String> kerberosConfiguration = kerberosConfigurations.get(ConfigHelper.fileNameToConfigType(filename)); + String type = ConfigHelper.fileNameToConfigType(filename); + String propertyName = propertyInfo.getName(); - if ((kerberosConfiguration != null) && (kerberosConfiguration.containsKey(propertyInfo.getName()))) { - kerberosConfiguration.put(propertyInfo.getName(), propertyInfo.getValue()); + Map<String, String> kerberosConfiguration = kerberosConfigurations.get(type); + if ((kerberosConfiguration != null) && (kerberosConfiguration.containsKey(propertyName))) { + kerberosConfiguration.put(propertyName, propertyInfo.getValue()); + } + + // Remove the relevant from the set of properties (for the given type) to remove + Collection<String> propertiesToRemove = configurationsToRemove.get(type); + if(propertiesToRemove != null) { + propertiesToRemove.remove(propertyName); } } } @@ -2417,9 +2438,25 @@ public class KerberosHelper { if (properties != null) { for (Map.Entry<String, String> configTypeEntry : properties.entrySet()) { + String value = configTypeEntry.getValue(); + kerberosConfDataFileBuilder.addRecord(type, configTypeEntry.getKey(), - configTypeEntry.getValue()); + value, + (value == null) ? KerberosConfigDataFile.OPERATION_TYPE_REMOVE : KerberosConfigDataFile.OPERATION_TYPE_SET + ); + } + } + } + + // Declare which properties to remove from the configurations + for (Map.Entry<String, Collection<String>> entry : configurationsToRemove.entrySet()) { + String type = entry.getKey(); + Collection<String> properties = entry.getValue(); + + if (properties != null) { + for (String propertyName : properties) { + kerberosConfDataFileBuilder.addRecord(type, propertyName, null, KerberosConfigDataFile.OPERATION_TYPE_REMOVE); } } } http://git-wip-us.apache.org/repos/asf/ambari/blob/8d3da8ef/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/KerberosConfigDataFile.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/KerberosConfigDataFile.java b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/KerberosConfigDataFile.java index db1a1d1..bbd0f66 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/KerberosConfigDataFile.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/KerberosConfigDataFile.java @@ -28,4 +28,8 @@ public class KerberosConfigDataFile { public static final String CONFIGURATION_TYPE = "config"; public static final String KEY = "key"; public static final String VALUE = "value"; + public static final String OPERATION = "operation"; + + public static final String OPERATION_TYPE_SET = "SET"; + public static final String OPERATION_TYPE_REMOVE = "REMOVE"; } http://git-wip-us.apache.org/repos/asf/ambari/blob/8d3da8ef/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/KerberosConfigDataFileBuilder.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/KerberosConfigDataFileBuilder.java b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/KerberosConfigDataFileBuilder.java index 20027bc..a10f38e 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/KerberosConfigDataFileBuilder.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/KerberosConfigDataFileBuilder.java @@ -49,17 +49,18 @@ public class KerberosConfigDataFileBuilder extends AbstractKerberosDataFileBuild /** * Appends a new record to the data file * - * @param config a String declaring the relevant configuration type for the key and value - * @param key a String declaring the key (or property name) with in the relevant configuration type - * @param value a String containing the value of the configuration property + * @param config a String declaring the relevant configuration type for the key and value + * @param key a String declaring the key (or property name) with in the relevant configuration type + * @param value a String containing the value of the configuration property + * @param operation a String containing the operation to perform, expected "SET" or "REMOVE" * @throws java.io.IOException */ - public void addRecord(String config, String key, String value) throws IOException { - super.appendRecord(config, key, value); + public void addRecord(String config, String key, String value, String operation) throws IOException { + super.appendRecord(config, key, value, operation); } @Override protected Iterable<String> getHeaderRecord() { - return Arrays.asList(CONFIGURATION_TYPE, KEY, VALUE); + return Arrays.asList(CONFIGURATION_TYPE, KEY, VALUE, OPERATION); } } http://git-wip-us.apache.org/repos/asf/ambari/blob/8d3da8ef/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/UpdateKerberosConfigsServerAction.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/UpdateKerberosConfigsServerAction.java b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/UpdateKerberosConfigsServerAction.java index eca9b79..9e342d0 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/UpdateKerberosConfigsServerAction.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/UpdateKerberosConfigsServerAction.java @@ -33,7 +33,9 @@ import org.slf4j.LoggerFactory; import java.io.File; import java.io.IOException; +import java.util.Collection; import java.util.HashMap; +import java.util.HashSet; import java.util.Map; import java.util.concurrent.ConcurrentMap; @@ -76,7 +78,8 @@ public class UpdateKerberosConfigsServerAction extends AbstractServerAction { String authenticatedUserName = getCommandParameterValue(getCommandParameters(), KerberosServerAction.AUTHENTICATED_USER_NAME); String dataDirectoryPath = getCommandParameterValue(getCommandParameters(), KerberosServerAction.DATA_DIRECTORY); - HashMap<String, Map<String, String>> configurations = new HashMap<String, Map<String, String>>(); + HashMap<String, Map<String, String>> propertiesToSet = new HashMap<String, Map<String, String>>(); + HashMap<String, Collection<String>> propertiesToRemove = new HashMap<String, Collection<String>>(); // If the data directory path is set, attempt to process further, else assume there is no work to do if (dataDirectoryPath != null) { @@ -101,7 +104,7 @@ public class UpdateKerberosConfigsServerAction extends AbstractServerAction { if (principalTokens.length == 2) { String principalConfigType = principalTokens[0]; String principalConfigProp = principalTokens[1]; - addConfigTypePropVal(configurations, principalConfigType, principalConfigProp, principal); + addConfigTypePropVal(propertiesToSet, principalConfigType, principalConfigProp, principal); } String keytabPath = record.get(KerberosActionDataFile.KEYTAB_FILE_PATH); @@ -110,7 +113,7 @@ public class UpdateKerberosConfigsServerAction extends AbstractServerAction { if (keytabTokens.length == 2) { String keytabConfigType = keytabTokens[0]; String keytabConfigProp = keytabTokens[1]; - addConfigTypePropVal(configurations, keytabConfigType, keytabConfigProp, keytabPath); + addConfigTypePropVal(propertiesToSet, keytabConfigType, keytabConfigProp, keytabPath); } } } @@ -124,17 +127,25 @@ public class UpdateKerberosConfigsServerAction extends AbstractServerAction { String configType = record.get(KerberosConfigDataFile.CONFIGURATION_TYPE); String configKey = record.get(KerberosConfigDataFile.KEY); String configVal = record.get(KerberosConfigDataFile.VALUE); - addConfigTypePropVal(configurations, configType, configKey, configVal); + String configOp = record.get(KerberosConfigDataFile.OPERATION); + + if (KerberosConfigDataFile.OPERATION_TYPE_REMOVE.equals(configOp)) { + removeConfigTypeProp(propertiesToRemove, configType, configKey); + } else { + addConfigTypePropVal(propertiesToSet, configType, configKey, configVal); + } } } - if (!configurations.isEmpty()) { + if (!propertiesToSet.isEmpty()) { String configNote = cluster.getSecurityType() == SecurityType.KERBEROS ? "Enabling Kerberos" : "Disabling Kerberos"; - for (Map.Entry<String, Map<String, String>> entry : configurations.entrySet()) { - configHelper.updateConfigType(cluster, controller, entry.getKey(), entry.getValue(), + for (Map.Entry<String, Map<String, String>> entry : propertiesToSet.entrySet()) { + String type = entry.getKey(); + + configHelper.updateConfigType(cluster, controller, type, entry.getValue(), propertiesToRemove.get(type), authenticatedUserName, configNote); } } @@ -185,18 +196,35 @@ public class UpdateKerberosConfigsServerAction extends AbstractServerAction { /** * Adds a property to properties of a given service config type * - * @param configurations - * @param configtype service config type + * @param configurations a map of configurations + * @param configType service config type * @param prop property to be added - * @param val value for the proeprty + * @param val value for the property */ - private void addConfigTypePropVal(HashMap<String, Map<String, String>> configurations, String configtype, String prop, String val) { - Map<String, String> configtypePropsVal = configurations.get(configtype); - if (configtypePropsVal == null) { - configtypePropsVal = new HashMap<String, String>(); - configurations.put(configtype, configtypePropsVal); + private void addConfigTypePropVal(HashMap<String, Map<String, String>> configurations, String configType, String prop, String val) { + Map<String, String> configTypePropsVal = configurations.get(configType); + if (configTypePropsVal == null) { + configTypePropsVal = new HashMap<String, String>(); + configurations.put(configType, configTypePropsVal); } - configtypePropsVal.put(prop, val); + configTypePropsVal.put(prop, val); + actionLog.writeStdOut(String.format("Setting property %s/%s: %s", configType, prop, (val == null) ? "<null>" : val)); } + /** + * Removes a property from the set of properties of a given service config type + * + * @param configurations a map of configurations + * @param configType service config type + * @param prop property to be removed + */ + private void removeConfigTypeProp(HashMap<String, Collection<String>> configurations, String configType, String prop) { + Collection<String> configTypeProps = configurations.get(configType); + if (configTypeProps == null) { + configTypeProps = new HashSet<String>(); + configurations.put(configType, configTypeProps); + } + configTypeProps.add(prop); + actionLog.writeStdOut(String.format("Removing property %s/%s", configType, prop)); + } } http://git-wip-us.apache.org/repos/asf/ambari/blob/8d3da8ef/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java index 83fca25..148e234 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java @@ -655,13 +655,15 @@ public class ConfigHelper { * @param controller * @param configType * @param updates + * @param removals a collection of property names to remove from the configuration type * @param authenticatedUserName * @param serviceVersionNote * @throws AmbariException */ public void updateConfigType(Cluster cluster, AmbariManagementController controller, String configType, - Map<String, String> updates, String authenticatedUserName, + Map<String, String> updates, Collection<String> removals, + String authenticatedUserName, String serviceVersionNote) throws AmbariException { if((configType != null) && (updates != null) && !updates.isEmpty()) { @@ -680,6 +682,13 @@ public class ConfigHelper { properties.putAll(updates); + // Remove properties that need to be removed. + if(removals != null) { + for (String propertyName : removals) { + properties.remove(propertyName); + } + } + if ((oldConfigProperties == null) || !Maps.difference(oldConfigProperties, properties).areEqual()) { createConfigType(cluster, controller, configType, properties, authenticatedUserName, serviceVersionNote); } http://git-wip-us.apache.org/repos/asf/ambari/blob/8d3da8ef/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/configuration/storm-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/configuration/storm-site.xml b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/configuration/storm-site.xml index bfe1d26..b6a2cf2 100644 --- a/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/configuration/storm-site.xml +++ b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/configuration/storm-site.xml @@ -192,11 +192,6 @@ <description>Childopts for Storm UI Java process.</description> </property> <property> - <name>ui.filter</name> - <value>null</value> - <description>Class for Storm UI authentication</description> - </property> - <property> <name>logviewer.port</name> <value>8000</value> <description>HTTP UI port for log viewer.</description> http://git-wip-us.apache.org/repos/asf/ambari/blob/8d3da8ef/ambari-server/src/test/java/org/apache/ambari/server/serveraction/kerberos/KerberosConfigDataFileTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/serveraction/kerberos/KerberosConfigDataFileTest.java b/ambari-server/src/test/java/org/apache/ambari/server/serveraction/kerberos/KerberosConfigDataFileTest.java index 51822cb..413de0b 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/serveraction/kerberos/KerberosConfigDataFileTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/serveraction/kerberos/KerberosConfigDataFileTest.java @@ -46,7 +46,10 @@ public class KerberosConfigDataFileTest { Assert.assertFalse(builder.isClosed()); for (int i = 0; i < 10; i++) { - builder.addRecord("config-type" + i, "key" + i, "value" + i); + builder.addRecord("config-type" + i, "key" + i, "value" + i, KerberosConfigDataFile.OPERATION_TYPE_SET); + } + for (int i = 10; i < 15; i++) { + builder.addRecord("config-type" + i, "key" + i, "value" + i, KerberosConfigDataFile.OPERATION_TYPE_REMOVE); } builder.close(); @@ -64,15 +67,24 @@ public class KerberosConfigDataFileTest { while (iterator.hasNext()) { Map<String, String> record = iterator.next(); - if (i < 10) { + if (i < 15) { Assert.assertEquals("config-type" + i, record.get(KerberosConfigDataFile.CONFIGURATION_TYPE)); Assert.assertEquals("key" + i, record.get(KerberosConfigDataFile.KEY)); Assert.assertEquals("value" + i, record.get(KerberosConfigDataFile.VALUE)); + + if(i<10) { + Assert.assertEquals("SET", record.get(KerberosConfigDataFile.OPERATION)); + } + else { + Assert.assertEquals("REMOVE", record.get(KerberosConfigDataFile.OPERATION)); + } } i++; } + Assert.assertEquals(15, i); + reader.close(); Assert.assertTrue(reader.isClosed()); reader.open(); @@ -89,6 +101,8 @@ public class KerberosConfigDataFileTest { i++; } + Assert.assertEquals(15, i); + reader.close(); Assert.assertTrue(reader.isClosed()); @@ -96,7 +110,7 @@ public class KerberosConfigDataFileTest { builder.open(); Assert.assertFalse(builder.isClosed()); - builder.addRecord("config-type", "key", "value"); + builder.addRecord("config-type", "key", "value", KerberosConfigDataFile.OPERATION_TYPE_SET); builder.close(); Assert.assertTrue(builder.isClosed()); @@ -109,7 +123,7 @@ public class KerberosConfigDataFileTest { i++; } - Assert.assertEquals(11, i); + Assert.assertEquals(16, i); reader.close(); Assert.assertTrue(reader.isClosed()); @@ -118,7 +132,7 @@ public class KerberosConfigDataFileTest { builder = new KerberosConfigDataFileBuilder(file); Assert.assertFalse(builder.isClosed()); - builder.addRecord("config-type", "key", "value"); + builder.addRecord("config-type", "key", "value", KerberosConfigDataFile.OPERATION_TYPE_REMOVE); builder.close(); Assert.assertTrue(builder.isClosed()); @@ -131,7 +145,7 @@ public class KerberosConfigDataFileTest { i++; } - Assert.assertEquals(12, i); + Assert.assertEquals(17, i); reader.close(); Assert.assertTrue(reader.isClosed()); http://git-wip-us.apache.org/repos/asf/ambari/blob/8d3da8ef/ambari-server/src/test/java/org/apache/ambari/server/serveraction/kerberos/UpdateKerberosConfigsServerActionTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/serveraction/kerberos/UpdateKerberosConfigsServerActionTest.java b/ambari-server/src/test/java/org/apache/ambari/server/serveraction/kerberos/UpdateKerberosConfigsServerActionTest.java index f902ba2..23ab519 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/serveraction/kerberos/UpdateKerberosConfigsServerActionTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/serveraction/kerberos/UpdateKerberosConfigsServerActionTest.java @@ -38,6 +38,7 @@ import org.junit.rules.TemporaryFolder; import java.io.BufferedWriter; import java.io.File; import java.io.FileWriter; +import java.util.Collection; import java.util.HashMap; import java.util.Map; import java.util.concurrent.ConcurrentMap; @@ -66,7 +67,7 @@ public class UpdateKerberosConfigsServerActionTest { replay(controller); configHelper.updateConfigType(anyObject(Cluster.class), anyObject(AmbariManagementController.class), - anyObject(String.class), anyObject(Map.class), anyObject(String.class), anyObject(String.class)); + anyObject(String.class), anyObject(Map.class), anyObject(Collection.class), anyObject(String.class), anyObject(String.class)); expectLastCall().atLeastOnce(); replay(configHelper);
