Updated Branches: refs/heads/trunk 72438ae29 -> 032055dcb
AMBARI-3142. Add support for task specific custom configuration. Project: http://git-wip-us.apache.org/repos/asf/incubator-ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-ambari/commit/032055dc Tree: http://git-wip-us.apache.org/repos/asf/incubator-ambari/tree/032055dc Diff: http://git-wip-us.apache.org/repos/asf/incubator-ambari/diff/032055dc Branch: refs/heads/trunk Commit: 032055dcb908358e82963a29856e89533551936b Parents: 72438ae Author: Sumit Mohanty <[email protected]> Authored: Mon Sep 16 19:52:09 2013 -0700 Committer: Sumit Mohanty <[email protected]> Committed: Mon Sep 16 19:52:09 2013 -0700 ---------------------------------------------------------------------- .../actionmanager/ExecutionCommandWrapper.java | 91 +++++++++++++------- .../server/configuration/Configuration.java | 2 + .../AmbariManagementControllerImpl.java | 25 +++--- .../AmbariManagementControllerTest.java | 46 +++++++++- 4 files changed, 115 insertions(+), 49 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/032055dc/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ExecutionCommandWrapper.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ExecutionCommandWrapper.java b/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ExecutionCommandWrapper.java index 182b7c6..cbce788 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ExecutionCommandWrapper.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ExecutionCommandWrapper.java @@ -33,10 +33,10 @@ import java.util.HashMap; import java.util.Map; public class ExecutionCommandWrapper { - private static Log LOG = LogFactory.getLog(ExecutionCommandWrapper.class); @Inject static Injector injector; - + private static Log LOG = LogFactory.getLog(ExecutionCommandWrapper.class); + private static String DELETED = "DELETED_"; String jsonExecutionCommand = null; ExecutionCommand executionCommand = null; @@ -48,59 +48,90 @@ public class ExecutionCommandWrapper { this.executionCommand = executionCommand; } + public static void applyCustomConfig(Map<String, Map<String, String>> configurations, String type, + String name, String value, Boolean deleted) { + if (!configurations.containsKey(type)) { + configurations.put(type, new HashMap<String, String>()); + } + String nameToUse = deleted ? DELETED + name : name; + Map<String, String> properties = configurations.get(type); + if (properties.containsKey(nameToUse)) { + properties.remove(nameToUse); + } + properties.put(nameToUse, value); + } + + public static Map<String, String> getMergedConfig(Map<String, String> persistedClusterConfig, + Map<String, String> override) { + Map<String, String> finalConfig = new HashMap<String, String>(persistedClusterConfig); + if (override != null && override.size() > 0) { + for (String overrideKey : override.keySet()) { + Boolean deleted = 0 == overrideKey.indexOf(DELETED); + String nameToUse = deleted ? overrideKey.substring(DELETED.length()) : overrideKey; + if (finalConfig.containsKey(nameToUse)) { + finalConfig.remove(nameToUse); + } + if (!deleted) { + finalConfig.put(nameToUse, override.get(overrideKey)); + } + } + } + + return finalConfig; + } + public ExecutionCommand getExecutionCommand() { if (executionCommand != null) { return executionCommand; } else if (jsonExecutionCommand != null) { -// try { executionCommand = StageUtils.getGson().fromJson(jsonExecutionCommand, ExecutionCommand.class); if (injector == null) { throw new RuntimeException("Injector not found, configuration cannot be restored"); - } else if ((executionCommand.getConfigurations() == null || executionCommand.getConfigurations().isEmpty()) && - executionCommand.getConfigurationTags() != null && + } else if (executionCommand.getConfigurationTags() != null && !executionCommand.getConfigurationTags().isEmpty()) { + // For a configuration type, both tag and an actual configuration can be stored + // Configurations from the tag is always expanded and then over-written by the actual + // global:version1:{a1:A1,b1:B1,d1:D1} + global:{a1:A2,c1:C1,DELETED_d1:x} ==> + // global:{a1:A2,b1:B1,c1:C1} Clusters clusters = injector.getInstance(Clusters.class); HostRoleCommandDAO hostRoleCommandDAO = injector.getInstance(HostRoleCommandDAO.class); - Long clusterId = hostRoleCommandDAO.findByPK(executionCommand.getTaskId()).getStage().getCluster().getClusterId(); + Long clusterId = hostRoleCommandDAO.findByPK( + executionCommand.getTaskId()).getStage().getCluster().getClusterId(); try { Cluster cluster = clusters.getClusterById(clusterId); - Map<String, Map<String, String>> configurations = new HashMap<String, Map<String, String>>(); - for (Map.Entry<String, Map<String, String>> entry : executionCommand.getConfigurationTags().entrySet()) { String type = entry.getKey(); Map<String, String> tags = entry.getValue(); - if (!configurations.containsKey(type)) { - configurations.put(type, new HashMap<String, String>()); - } - - String tag; - - //perform override //TODO align with configs override logic - tag = tags.get("host_override_tag"); + String tag = tags.get("host_override_tag"); tag = tag == null ? tags.get("service_override_tag") : tag; tag = tag == null ? tags.get("tag") : tag; if (tag != null) { Config config = cluster.getConfig(type, tag); - configurations.get(type).putAll(config.getProperties()); + if (executionCommand.getConfigurations().containsKey(type)) { + Map<String, String> mergedConfig = + getMergedConfig(config.getProperties(), executionCommand.getConfigurations().get(type)); + executionCommand.getConfigurations().get(type).clear(); + executionCommand.getConfigurations().get(type).putAll(mergedConfig); + + } else { + executionCommand.getConfigurations().put(type, new HashMap<String, String>()); + executionCommand.getConfigurations().get(type).putAll(config.getProperties()); + } } } - executionCommand.setConfigurations(configurations); } catch (AmbariException e) { throw new RuntimeException(e); } } return executionCommand; -// } catch (IOException e) { -// throw new RuntimeException(e); -// } } else { throw new RuntimeException( "Invalid ExecutionCommandWrapper, both object and string" @@ -112,14 +143,8 @@ public class ExecutionCommandWrapper { if (jsonExecutionCommand != null) { return jsonExecutionCommand; } else if (executionCommand != null) { -// try { - jsonExecutionCommand = StageUtils.getGson().toJson(executionCommand); - return jsonExecutionCommand; -// } catch (JAXBException e) { -// throw new RuntimeException(e); -// } catch (IOException e) { -// throw new RuntimeException(e); -// } + jsonExecutionCommand = StageUtils.getGson().toJson(executionCommand); + return jsonExecutionCommand; } else { throw new RuntimeException( "Invalid ExecutionCommandWrapper, both object and string" @@ -129,13 +154,15 @@ public class ExecutionCommandWrapper { @Override public boolean equals(Object o) { - if (this == o) + if (this == o) { return true; - if (o == null || getClass() != o.getClass()) + } + if (o == null || getClass() != o.getClass()) { return false; + } ExecutionCommandWrapper wrapper = (ExecutionCommandWrapper) o; - + if (executionCommand != null && wrapper.executionCommand != null) { return executionCommand.equals(wrapper.executionCommand); } else { http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/032055dc/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java b/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java index 04828f0..4364a79 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java @@ -235,6 +235,8 @@ public class Configuration { private static final String SERVER_CONNECTION_MAX_IDLE_TIME = "server.connection.max.idle.millis"; + public static final String GLOBAL_CONFIG_TAG = "global"; + public static final String RCA_ENABLED_PROPERTY = "rca_enabled"; public static final String HIVE_CONFIG_TAG = "hive-site"; public static final String HIVE_METASTORE_PASSWORD_PROPERTY = "javax.jdo.option.ConnectionPassword"; http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/032055dc/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java index a280907..dd37821 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java @@ -45,6 +45,7 @@ import org.apache.ambari.server.ServiceComponentNotFoundException; import org.apache.ambari.server.ServiceNotFoundException; import org.apache.ambari.server.StackAccessException; import org.apache.ambari.server.actionmanager.ActionManager; +import org.apache.ambari.server.actionmanager.ExecutionCommandWrapper; import org.apache.ambari.server.actionmanager.HostRoleCommand; import org.apache.ambari.server.actionmanager.HostRoleStatus; import org.apache.ambari.server.actionmanager.RequestStatus; @@ -976,11 +977,8 @@ public class AmbariManagementControllerImpl implements // Hack - Remove passwords from configs if (event.getServiceComponentName().equals(Role.HIVE_CLIENT.toString())) { - Map<String, String> hiveConfigs = configurations.get(Configuration - .HIVE_CONFIG_TAG); - if (hiveConfigs != null) { - hiveConfigs.remove(Configuration.HIVE_METASTORE_PASSWORD_PROPERTY); - } + ExecutionCommandWrapper.applyCustomConfig(configurations, Configuration.HIVE_CONFIG_TAG, + Configuration.HIVE_METASTORE_PASSWORD_PROPERTY, "", true); } execCmd.setConfigurations(configurations); @@ -2053,7 +2051,7 @@ public class AmbariManagementControllerImpl implements RoleCommand roleCommand; State oldSchState = scHost.getState(); ServiceComponentHostEvent event; - switch(newState) { + switch (newState) { case INSTALLED: if (oldSchState == State.INIT || oldSchState == State.UNINSTALLED @@ -2162,20 +2160,19 @@ public class AmbariManagementControllerImpl implements } // [ type -> [ key, value ] ] - Map<String, Map<String, String>> configurations = new TreeMap<String, Map<String,String>>(); - Map<String, Map<String, String>> configTags = new HashMap<String, Map<String,String>>(); + Map<String, Map<String, String>> configurations = new TreeMap<String, Map<String, String>>(); + Map<String, Map<String, String>> configTags = new HashMap<String, Map<String, String>>(); findConfigurationPropertiesWithOverrides(configurations, configTags, cluster, scHost.getServiceName(), scHost.getHostName(), clusterDesiredConfigs, configsByHosts.get(scHost.getHostName())); // HACK HACK HACK - if ((!scHost.getHostName().equals(jobtrackerHost)) - && configurations.get("global") != null) { - if (LOG.isDebugEnabled()) { - LOG.debug("Setting rca_enabled to false for host " - + scHost.getHostName()); + if (!scHost.getHostName().equals(jobtrackerHost)) { + if (configTags.get(Configuration.GLOBAL_CONFIG_TAG) != null) { + ExecutionCommandWrapper.applyCustomConfig( + configurations, Configuration.GLOBAL_CONFIG_TAG, + Configuration.RCA_ENABLED_PROPERTY, "false", false); } - configurations.get("global").put("rca_enabled", "false"); } createHostAction(cluster, stage, scHost, configurations, configTags, http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/032055dc/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java index 6336bfe..d2bf1e6 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java @@ -5333,13 +5333,15 @@ public class AmbariManagementControllerTest { // Install installService(clusterName, serviceName, false, false); // Start - long requestId = startService(clusterName, serviceName, false, false); + long requestId = startService(clusterName, serviceName, false, true); String passwordInConfig = null; + Boolean isClientInstalled = false; List<Stage> stages = actionDB.getAllStages(requestId); for (Stage s : stages) { for (HostRoleCommand hrc : s.getOrderedHostRoleCommands()) { - if (hrc.getRole().equals(Role.HIVE_CLIENT.toString())) { + if (hrc.getRole().equals(Role.HIVE_CLIENT)) { + isClientInstalled = true; Map<String, String> hiveSite = hrc.getExecutionCommandWrapper() .getExecutionCommand().getConfigurations().get("hive-site"); Assert.assertNotNull(hiveSite); @@ -5349,6 +5351,7 @@ public class AmbariManagementControllerTest { } } } + Assert.assertTrue("HIVE_CLIENT must be installed", isClientInstalled); Assert.assertNull(passwordInConfig); } @@ -7219,7 +7222,44 @@ public class AmbariManagementControllerTest { controller.deleteHostComponents(schRequests); Assert.assertEquals(0, cluster.getServiceComponentHosts(host1).size()); - } + } + + @Test + public void testExecutionCommandConfiguration() throws AmbariException { + Map<String, Map<String, String>> config = new HashMap<String, Map<String, String>>(); + config.put("type1", new HashMap<String, String>()); + config.put("type3", new HashMap<String, String>()); + config.get("type3").put("name1", "neverchange"); + ExecutionCommandWrapper.applyCustomConfig(config, "type1", "name1", "value11", false); + Assert.assertEquals("value11", config.get("type1").get("name1")); + + config.put("type1", new HashMap<String, String>()); + ExecutionCommandWrapper.applyCustomConfig(config, "type1", "name1", "value12", false); + Assert.assertEquals("value12", config.get("type1").get("name1")); + + ExecutionCommandWrapper.applyCustomConfig(config, "type2", "name2", "value21", false); + Assert.assertEquals("value21", config.get("type2").get("name2")); + + ExecutionCommandWrapper.applyCustomConfig(config, "type2", "name2", "", true); + Assert.assertEquals("", config.get("type2").get("DELETED_name2")); + Assert.assertEquals("neverchange", config.get("type3").get("name1")); + + Map<String, String> persistedClusterConfig = new HashMap<String, String>(); + persistedClusterConfig.put("name1", "value11"); + persistedClusterConfig.put("name3", "value31"); + persistedClusterConfig.put("name4", "value41"); + Map<String, String> override = new HashMap<String, String>(); + override.put("name1", "value12"); + override.put("name2", "value21"); + override.put("DELETED_name3", "value31"); + Map<String, String> mergedConfig = ExecutionCommandWrapper.getMergedConfig(persistedClusterConfig, + override); + Assert.assertEquals(3, mergedConfig.size()); + Assert.assertFalse(mergedConfig.containsKey("name3")); + Assert.assertEquals("value12", mergedConfig.get("name1")); + Assert.assertEquals("value21", mergedConfig.get("name2")); + Assert.assertEquals("value41", mergedConfig.get("name4")); + } } \ No newline at end of file
