Repository: ambari Updated Branches: refs/heads/trunk cabf68299 -> 3ac13b5c7
AMBARI-17836 - EU POST Call Takes More Than 3 Minutes on 900 Node Cluster (jonathanhurley) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/3ac13b5c Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/3ac13b5c Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/3ac13b5c Branch: refs/heads/trunk Commit: 3ac13b5c793a05da245722f122a6445af2ba1169 Parents: cabf682 Author: Jonathan Hurley <[email protected]> Authored: Thu Jul 21 12:36:12 2016 -0400 Committer: Jonathan Hurley <[email protected]> Committed: Fri Jul 22 09:29:57 2016 -0400 ---------------------------------------------------------------------- .../src/main/python/ambari_agent/ActionQueue.py | 10 - .../server/actionmanager/ActionScheduler.java | 3 +- .../actionmanager/ExecutionCommandWrapper.java | 210 ++++++++++++------- .../ambari/server/agent/ExecutionCommand.java | 32 +-- .../server/configuration/Configuration.java | 2 +- .../controller/AmbariActionExecutionHelper.java | 40 ++-- .../AmbariCustomCommandExecutionHelper.java | 184 ++++++++-------- .../AmbariManagementControllerImpl.java | 8 +- .../internal/ClientConfigResourceProvider.java | 110 +++++----- .../internal/UpgradeResourceProvider.java | 2 +- .../ambari/server/state/ConfigHelper.java | 8 +- .../ambari/server/state/UpgradeHelper.java | 20 +- .../AmbariCustomCommandExecutionHelperTest.java | 8 +- .../AmbariManagementControllerImplTest.java | 15 +- .../ClientConfigResourceProviderTest.java | 78 +++++-- .../UpgradeResourceProviderHDP22Test.java | 119 ++--------- 16 files changed, 422 insertions(+), 427 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/3ac13b5c/ambari-agent/src/main/python/ambari_agent/ActionQueue.py ---------------------------------------------------------------------- diff --git a/ambari-agent/src/main/python/ambari_agent/ActionQueue.py b/ambari-agent/src/main/python/ambari_agent/ActionQueue.py index 1e7b1b6..c8696ef 100644 --- a/ambari-agent/src/main/python/ambari_agent/ActionQueue.py +++ b/ambari-agent/src/main/python/ambari_agent/ActionQueue.py @@ -407,16 +407,6 @@ class ActionQueue(threading.Thread): # let ambari know that configuration tags were applied configHandler = ActualConfigHandler(self.config, self.configTags) - #update - if command.has_key('forceRefreshConfigTags') and len(command['forceRefreshConfigTags']) > 0 : - forceRefreshConfigTags = command['forceRefreshConfigTags'] - logger.info("Got refresh additional component tags command") - - for configTag in forceRefreshConfigTags : - configHandler.update_component_tag(command['role'], configTag, command['configurationTags'][configTag]) - - roleResult['customCommand'] = self.CUSTOM_COMMAND_RESTART # force restart for component to evict stale_config on server side - command['configurationTags'] = configHandler.read_actual_component(command['role']) if command.has_key('configurationTags'): configHandler.write_actual(command['configurationTags']) http://git-wip-us.apache.org/repos/asf/ambari/blob/3ac13b5c/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ActionScheduler.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ActionScheduler.java b/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ActionScheduler.java index 205ef9f..90ffa5b 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ActionScheduler.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ActionScheduler.java @@ -523,8 +523,7 @@ class ActionScheduler implements Runnable { private boolean isStageHasBackgroundCommandsOnly(Stage s, String host) { for (ExecutionCommandWrapper c : s.getExecutionCommands(host)) { - if(c.getExecutionCommand().getCommandType() != AgentCommandType.BACKGROUND_EXECUTION_COMMAND) - { + if (c.getCommandType() != AgentCommandType.BACKGROUND_EXECUTION_COMMAND) { return false; } } http://git-wip-us.apache.org/repos/asf/ambari/blob/3ac13b5c/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ExecutionCommandWrapper.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ExecutionCommandWrapper.java b/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ExecutionCommandWrapper.java index ef12c3a..2f6ab02 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ExecutionCommandWrapper.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ExecutionCommandWrapper.java @@ -23,17 +23,17 @@ import java.util.Set; import java.util.TreeMap; import org.apache.ambari.server.AmbariException; +import org.apache.ambari.server.ClusterNotFoundException; +import org.apache.ambari.server.agent.AgentCommand.AgentCommandType; import org.apache.ambari.server.agent.ExecutionCommand; -import org.apache.ambari.server.controller.AmbariManagementController; -import org.apache.ambari.server.controller.AmbariServer; import org.apache.ambari.server.orm.dao.HostRoleCommandDAO; import org.apache.ambari.server.state.Cluster; import org.apache.ambari.server.state.Clusters; import org.apache.ambari.server.state.ConfigHelper; -import org.apache.ambari.server.utils.StageUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.google.gson.Gson; import com.google.inject.Inject; import com.google.inject.assistedinject.Assisted; import com.google.inject.assistedinject.AssistedInject; @@ -54,6 +54,9 @@ public class ExecutionCommandWrapper { @Inject ConfigHelper configHelper; + @Inject + private Gson gson; + @AssistedInject public ExecutionCommandWrapper(@Assisted String jsonExecutionCommand) { this.jsonExecutionCommand = jsonExecutionCommand; @@ -64,110 +67,155 @@ public class ExecutionCommandWrapper { this.executionCommand = executionCommand; } - @SuppressWarnings("serial") + /** + * Gets the execution command by either de-serializing the backing JSON + * command or returning the encapsulated instance which has already been + * de-serialized. + * <p/> + * If the {@link ExecutionCommand} has configuration tags which need to be + * refreshed, then this method will lookup the appropriate configuration tags + * before building the final configurations to set ont he command. Therefore, + * the {@link ExecutionCommand} is allowed to have no configuration tags as + * long as it has been instructed to set updated ones at execution time. + * + * @return + * @see ExecutionCommand#setForceRefreshConfigTagsBeforeExecution(Set) + */ public ExecutionCommand getExecutionCommand() { if (executionCommand != null) { return executionCommand; - } else if (jsonExecutionCommand != null) { - executionCommand = StageUtils.getGson().fromJson(jsonExecutionCommand, ExecutionCommand.class); - - if (executionCommand.getConfigurationTags() != null - && - !executionCommand.getConfigurationTags().isEmpty()) { - - // For a configuration type, both tag and an actual configuration can be stored - // Configurations from the tag is always expanded and then over-written by the actual - // global:version1:{a1:A1,b1:B1,d1:D1} + global:{a1:A2,c1:C1,DELETED_d1:x} ==> - // global:{a1:A2,b1:B1,c1:C1} - Long clusterId = hostRoleCommandDAO.findByPK( - executionCommand.getTaskId()).getStage().getClusterId(); - - try { - Cluster cluster = clusters.getClusterById(clusterId); - Map<String, Map<String, String>> configurationTags = executionCommand.getConfigurationTags(); - - // Execution commands have config-tags already set during their creation. However, these - // tags become stale at runtime when other ExecutionCommands run and change the desired - // configs (like ConfigureAction). Hence an ExecutionCommand can specify which config-types - // should be refreshed at runtime. Specifying <code>*</code> will result in all config-type - // tags to be refreshed to the latest cluster desired-configs. - Set<String> refreshConfigTagsBeforeExecution = executionCommand.getForceRefreshConfigTagsBeforeExecution(); - if (refreshConfigTagsBeforeExecution != null && !refreshConfigTagsBeforeExecution.isEmpty()) { - AmbariManagementController managementController = AmbariServer.getController(); - Map<String, Map<String, String>> configTags = managementController.findConfigurationTagsWithOverrides( - cluster, executionCommand.getHostname()); - for (String refreshConfigTag : refreshConfigTagsBeforeExecution) { - if ("*".equals(refreshConfigTag)) { - // if forcing a refresh of *, then clear out any existing - // configurations so that all of the new configurations are - // forcefully applied - LOG.debug("ExecutionCommandWrapper.getExecutionCommand: refreshConfigTag set to {}, so clearing config for full refresh.", refreshConfigTag); - executionCommand.getConfigurations().clear(); - configurationTags = configTags; - executionCommand.setConfigurationTags(configTags); - break; - } else if (configurationTags.containsKey(refreshConfigTag) && configTags.containsKey(refreshConfigTag)) { - configurationTags.put(refreshConfigTag, configTags.get(refreshConfigTag)); - } - } - } + } + + if( null == jsonExecutionCommand ){ + throw new RuntimeException( + "Invalid ExecutionCommandWrapper, both object and string" + + " representations are null"); + } + + try { + executionCommand = gson.fromJson(jsonExecutionCommand, ExecutionCommand.class); + + // sanity; if no configurations, just initialize to prevent NPEs + if (null == executionCommand.getConfigurations()) { + executionCommand.setConfigurations(new TreeMap<String, Map<String, String>>()); + } + + Map<String, Map<String, String>> configurations = executionCommand.getConfigurations(); + + // For a configuration type, both tag and an actual configuration can be stored + // Configurations from the tag is always expanded and then over-written by the actual + // global:version1:{a1:A1,b1:B1,d1:D1} + global:{a1:A2,c1:C1,DELETED_d1:x} ==> + // global:{a1:A2,b1:B1,c1:C1} + Long clusterId = hostRoleCommandDAO.findByPK( + executionCommand.getTaskId()).getStage().getClusterId(); + + Cluster cluster = clusters.getClusterById(clusterId); + + // Execution commands may have config-tags already set during their creation. + // However, these tags become stale at runtime when other + // ExecutionCommands run and change the desired configs (like + // ConfigureAction). Hence an ExecutionCommand can specify which + // config-types should be refreshed at runtime. Specifying <code>*</code> + // will result in all config-type tags to be refreshed to the latest + // cluster desired-configs. Additionally, there may be no configuration + // tags set but refresh might be set to *. In this case, they should still + // be refreshed with the latest. + boolean refreshConfigTagsBeforeExecution = executionCommand.getForceRefreshConfigTagsBeforeExecution(); + if (refreshConfigTagsBeforeExecution) { + Map<String, Map<String, String>> configurationTags = configHelper.getEffectiveDesiredTags( + cluster, executionCommand.getHostname()); + + // then clear out any existing configurations so that all of the new + // configurations are forcefully applied + LOG.debug("Refreshing all configuration tags before execution"); - Map<String, Map<String, String>> configProperties = configHelper + configurations.clear(); + executionCommand.setConfigurationTags(configurationTags); + } + + // now that the tags have been updated (if necessary), fetch the + // configurations + Map<String, Map<String, String>> configurationTags = executionCommand.getConfigurationTags(); + if (null != configurationTags && !configurationTags.isEmpty()) { + Map<String, Map<String, String>> configProperties = configHelper .getEffectiveConfigProperties(cluster, configurationTags); - // Apply the configurations saved with the Execution Cmd on top of - // derived configs - This will take care of all the hacks - for (Map.Entry<String, Map<String, String>> entry : configProperties.entrySet()) { - String type = entry.getKey(); - Map<String, String> allLevelMergedConfig = entry.getValue(); + // Apply the configurations saved with the Execution Cmd on top of + // derived configs - This will take care of all the hacks + for (Map.Entry<String, Map<String, String>> entry : configProperties.entrySet()) { + String type = entry.getKey(); + Map<String, String> allLevelMergedConfig = entry.getValue(); - if (executionCommand.getConfigurations().containsKey(type)) { - Map<String, String> mergedConfig = - configHelper.getMergedConfig(allLevelMergedConfig, - executionCommand.getConfigurations().get(type)); + if (configurations.containsKey(type)) { + Map<String, String> mergedConfig = configHelper.getMergedConfig(allLevelMergedConfig, + configurations.get(type)); - executionCommand.getConfigurations().get(type).clear(); - executionCommand.getConfigurations().get(type).putAll(mergedConfig); + configurations.get(type).clear(); + configurations.get(type).putAll(mergedConfig); - } else { - executionCommand.getConfigurations().put(type, new HashMap<String, String>()); - executionCommand.getConfigurations().get(type).putAll(allLevelMergedConfig); - } + } else { + configurations.put(type, new HashMap<String, String>()); + configurations.get(type).putAll(allLevelMergedConfig); } + } - Map<String, Map<String, Map<String, String>>> configAttributes = configHelper.getEffectiveConfigAttributes(cluster, - executionCommand.getConfigurationTags()); + Map<String, Map<String, Map<String, String>>> configAttributes = configHelper.getEffectiveConfigAttributes( + cluster, executionCommand.getConfigurationTags()); - for (Map.Entry<String, Map<String, Map<String, String>>> attributesOccurrence : configAttributes.entrySet()) { - String type = attributesOccurrence.getKey(); - Map<String, Map<String, String>> attributes = attributesOccurrence.getValue(); + for (Map.Entry<String, Map<String, Map<String, String>>> attributesOccurrence : configAttributes.entrySet()) { + String type = attributesOccurrence.getKey(); + Map<String, Map<String, String>> attributes = attributesOccurrence.getValue(); - if (executionCommand.getConfigurationAttributes() != null) { - if (!executionCommand.getConfigurationAttributes().containsKey(type)) { - executionCommand.getConfigurationAttributes().put(type, new TreeMap<String, Map<String, String>>()); - } - configHelper.cloneAttributesMap(attributes, executionCommand.getConfigurationAttributes().get(type)); + if (executionCommand.getConfigurationAttributes() != null) { + if (!executionCommand.getConfigurationAttributes().containsKey(type)) { + executionCommand.getConfigurationAttributes().put(type, + new TreeMap<String, Map<String, String>>()); + } + configHelper.cloneAttributesMap(attributes, + executionCommand.getConfigurationAttributes().get(type)); } - } - - } catch (AmbariException e) { - throw new RuntimeException(e); } } + } catch (ClusterNotFoundException cnfe) { + // it's possible that there are commands without clusters; in such cases, + // just return the de-serialized command and don't try to read configs + LOG.warn( + "Unable to lookup the cluster byt ID; assuming that there is no cluster and therefore no configs for this execution command: {}", + cnfe.getMessage()); return executionCommand; - } else { + } catch (AmbariException e) { + throw new RuntimeException(e); + } + + return executionCommand; + } + + /** + * Gets the type of command by deserializing the JSON and invoking + * {@link ExecutionCommand#getCommandType()}. + * + * @return + */ + public AgentCommandType getCommandType() { + if (executionCommand != null) { + return executionCommand.getCommandType(); + } + + if (null == jsonExecutionCommand) { throw new RuntimeException( - "Invalid ExecutionCommandWrapper, both object and string" - + " representations are null"); + "Invalid ExecutionCommandWrapper, both object and string" + " representations are null"); } + + return gson.fromJson(jsonExecutionCommand, + ExecutionCommand.class).getCommandType(); } public String getJson() { if (jsonExecutionCommand != null) { return jsonExecutionCommand; } else if (executionCommand != null) { - jsonExecutionCommand = StageUtils.getGson().toJson(executionCommand); + jsonExecutionCommand = gson.toJson(executionCommand); return jsonExecutionCommand; } else { throw new RuntimeException( http://git-wip-us.apache.org/repos/asf/ambari/blob/3ac13b5c/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java b/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java index 28de8ed..29737ee 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java @@ -88,11 +88,8 @@ public class ExecutionCommand extends AgentCommand { @SerializedName("configurationTags") private Map<String, Map<String, String>> configurationTags; - @SerializedName("forceRefreshConfigTags") - private Set<String> forceRefreshConfigTags = new HashSet<String>(); - @SerializedName("forceRefreshConfigTagsBeforeExecution") - private Set<String> forceRefreshConfigTagsBeforeExecution = new HashSet<String>(); + private boolean forceRefreshConfigTagsBeforeExecution = false; @SerializedName("commandParams") private Map<String, String> commandParams = new HashMap<String, String>(); @@ -233,27 +230,16 @@ public class ExecutionCommand extends AgentCommand { public void setConfigurations(Map<String, Map<String, String>> configurations) { this.configurations = configurations; } - /** - * @return Returns the set of config-types that have to be propagated to actual-config of component of given custom command, if command is successfully finished. - */ - public Set<String> getForceRefreshConfigTags() { - return forceRefreshConfigTags; - } - - public void setForceRefreshConfigTags(Set<String> forceRefreshConfigTags) { - this.forceRefreshConfigTags = forceRefreshConfigTags; - } /** - * Comma separated list of config-types whose tags have be refreshed - * at runtime before being executed. If all config-type tags have to be - * refreshed, "*" can be specified. + * Gets whether configuration tags shoudl be refreshed right before the + * command is scheduled. */ - public Set<String> getForceRefreshConfigTagsBeforeExecution() { + public boolean getForceRefreshConfigTagsBeforeExecution() { return forceRefreshConfigTagsBeforeExecution; } - public void setForceRefreshConfigTagsBeforeExecution(Set<String> forceRefreshConfigTagsBeforeExecution) { + public void setForceRefreshConfigTagsBeforeExecution(boolean forceRefreshConfigTagsBeforeExecution) { this.forceRefreshConfigTagsBeforeExecution = forceRefreshConfigTagsBeforeExecution; } @@ -274,7 +260,7 @@ public class ExecutionCommand extends AgentCommand { for (Map.Entry<String, ServiceInfo> entry : serviceInfoMap.entrySet()) { serviceVersionMap.put(entry.getKey(), entry.getValue().getVersion()); } - this.availableServices = serviceVersionMap; + availableServices = serviceVersionMap; } public Map<String, Map<String, Map<String, String>>> getConfigurationAttributes() { @@ -379,7 +365,6 @@ public class ExecutionCommand extends AgentCommand { String AMBARI_DB_RCA_USERNAME = "ambari_db_rca_username"; String AMBARI_DB_RCA_PASSWORD = "ambari_db_rca_password"; String COMPONENT_CATEGORY = "component_category"; - String REFRESH_ADITIONAL_COMPONENT_TAGS = "forceRefreshConfigTags"; String USER_LIST = "user_list"; String GROUP_LIST = "group_list"; String NOT_MANAGED_HDFS_PATH_LIST = "not_managed_hdfs_path_list"; @@ -393,9 +378,8 @@ public class ExecutionCommand extends AgentCommand { String LOG_OUTPUT = "log_output"; /** - * Comma separated list of config-types whose tags have be refreshed - * at runtime before being executed. If all config-type tags have to be - * refreshed, "*" can be specified. + * A boolean indicating whether configuration tags should be refreshed + * before sending the command. */ String REFRESH_CONFIG_TAGS_BEFORE_EXECUTION = "forceRefreshConfigTagsBeforeExecution"; http://git-wip-us.apache.org/repos/asf/ambari/blob/3ac13b5c/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java b/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java index 399f26c..9655bf9 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java @@ -2634,7 +2634,7 @@ public class Configuration { * @return the location of the template file, or {@code null} if not defined. */ public String getAlertTemplateFile() { - return properties.getProperty(ALERT_TEMPLATE_FILE); + return StringUtils.strip(properties.getProperty(ALERT_TEMPLATE_FILE)); } /** http://git-wip-us.apache.org/repos/asf/ambari/blob/3ac13b5c/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariActionExecutionHelper.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariActionExecutionHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariActionExecutionHelper.java index 9c2db1c..5cc807c 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariActionExecutionHelper.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariActionExecutionHelper.java @@ -28,7 +28,6 @@ import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.SCRIPT_TY import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.STACK_NAME; import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.STACK_VERSION; -import java.util.Arrays; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -369,6 +368,10 @@ public class AmbariActionExecutionHelper { // create tasks for each host for (String hostName : targetHosts) { + // ensure that any tags that need to be refreshed are extracted from the + // context and put onto the execution command + Map<String, String> actionParameters = actionContext.getParameters(); + stage.addHostRoleExecutionCommand(hostName, Role.valueOf(actionContext.getActionName()), RoleCommand.ACTIONEXECUTE, new ServiceComponentHostOpInProgressEvent(actionContext.getActionName(), hostName, @@ -401,10 +404,22 @@ public class AmbariActionExecutionHelper { execCmd.setConfigurations(new TreeMap<String, Map<String, String>>()); execCmd.setConfigurationAttributes(new TreeMap<String, Map<String, Map<String, String>>>()); - // !!! ensure that the config tags are added to this command so that the - // configurations can be populated from the tags before the command is - // sent - Map<String, Map<String, String>> configTags = managementController.findConfigurationTagsWithOverrides(cluster, hostName); + // if the command should fetch brand new configuration tags before + // execution, then we don't need to fetch them now + if (null != actionParameters && !actionParameters.isEmpty()) { + if (actionParameters.containsKey(KeyNames.REFRESH_CONFIG_TAGS_BEFORE_EXECUTION)) { + execCmd.setForceRefreshConfigTagsBeforeExecution(true); + } + } + + // when building complex orchestration ahead of time (such as when + // performing ugprades), fetching configuration tags can take a very long + // time - if it's not needed, then don't do it + Map<String, Map<String, String>> configTags = new TreeMap<String, Map<String, String>>(); + if (!execCmd.getForceRefreshConfigTagsBeforeExecution()) { + configTags = managementController.findConfigurationTagsWithOverrides(cluster, hostName); + } + execCmd.setConfigurationTags(configTags); execCmd.setCommandParams(commandParams); @@ -431,7 +446,7 @@ public class AmbariActionExecutionHelper { roleParams = new TreeMap<String, String>(); } - roleParams.putAll(actionContext.getParameters()); + roleParams.putAll(actionParameters); SecretReference.replaceReferencesWithPasswords(roleParams, cluster); @@ -447,19 +462,6 @@ public class AmbariActionExecutionHelper { execCmd.setRoleParams(roleParams); - // ensure that any tags that need to be refreshed are extracted from the - // context and put onto the execution command - Map<String, String> actionParameters = actionContext.getParameters(); - if (null != actionParameters && !actionParameters.isEmpty()) { - if (actionParameters.containsKey(KeyNames.REFRESH_CONFIG_TAGS_BEFORE_EXECUTION)) { - String[] split = StringUtils.split( - actionParameters.get(KeyNames.REFRESH_CONFIG_TAGS_BEFORE_EXECUTION)); - Set<String> configsToRefresh = new HashSet<String>(Arrays.asList(split)); - - execCmd.setForceRefreshConfigTagsBeforeExecution(configsToRefresh); - } - } - if (null != cluster) { // Generate localComponents for (ServiceComponentHost sch : cluster.getServiceComponentHosts(hostName)) { http://git-wip-us.apache.org/repos/asf/ambari/blob/3ac13b5c/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java index 2174a64..8c8ae10 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java @@ -18,6 +18,33 @@ package org.apache.ambari.server.controller; +import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.AGENT_STACK_RETRY_COUNT; +import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.AGENT_STACK_RETRY_ON_UNAVAILABILITY; +import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.CLIENTS_TO_UPDATE_CONFIGS; +import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.COMMAND_TIMEOUT; +import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.COMPONENT_CATEGORY; +import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.CUSTOM_COMMAND; +import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.DB_DRIVER_FILENAME; +import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.DB_NAME; +import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.GROUP_LIST; +import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.HOOKS_FOLDER; +import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.HOST_SYS_PREPPED; +import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.JAVA_HOME; +import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.JAVA_VERSION; +import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.JCE_NAME; +import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.JDK_LOCATION; +import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.JDK_NAME; +import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.MYSQL_JDBC_URL; +import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.NOT_MANAGED_HDFS_PATH_LIST; +import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.ORACLE_JDBC_URL; +import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.REPO_INFO; +import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.SCRIPT; +import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.SCRIPT_TYPE; +import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.SERVICE_PACKAGE_FOLDER; +import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.STACK_NAME; +import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.STACK_VERSION; +import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.USER_LIST; + import java.text.MessageFormat; import java.util.ArrayList; import java.util.Collections; @@ -54,6 +81,7 @@ import org.apache.ambari.server.state.CommandScriptDefinition; import org.apache.ambari.server.state.ComponentInfo; import org.apache.ambari.server.state.ConfigHelper; import org.apache.ambari.server.state.CustomCommandDefinition; +import org.apache.ambari.server.state.DesiredConfig; import org.apache.ambari.server.state.Host; import org.apache.ambari.server.state.HostComponentAdminState; import org.apache.ambari.server.state.HostState; @@ -83,33 +111,6 @@ import com.google.gson.JsonObject; import com.google.inject.Inject; import com.google.inject.Singleton; -import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.AGENT_STACK_RETRY_COUNT; -import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.AGENT_STACK_RETRY_ON_UNAVAILABILITY; -import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.CLIENTS_TO_UPDATE_CONFIGS; -import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.COMMAND_TIMEOUT; -import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.COMPONENT_CATEGORY; -import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.CUSTOM_COMMAND; -import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.DB_DRIVER_FILENAME; -import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.DB_NAME; -import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.GROUP_LIST; -import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.HOOKS_FOLDER; -import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.HOST_SYS_PREPPED; -import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.JAVA_HOME; -import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.JAVA_VERSION; -import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.JCE_NAME; -import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.JDK_LOCATION; -import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.JDK_NAME; -import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.MYSQL_JDBC_URL; -import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.NOT_MANAGED_HDFS_PATH_LIST; -import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.ORACLE_JDBC_URL; -import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.REPO_INFO; -import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.SCRIPT; -import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.SCRIPT_TYPE; -import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.SERVICE_PACKAGE_FOLDER; -import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.STACK_NAME; -import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.STACK_VERSION; -import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.USER_LIST; - /** * Helper class containing logic to process custom command execution requests . * This class has special support needed for SERVICE_CHECK and DECOMMISSION. @@ -338,8 +339,23 @@ public class AmbariCustomCommandExecutionHelper { new TreeMap<String, Map<String, String>>(); Map<String, Map<String, Map<String, String>>> configurationAttributes = new TreeMap<String, Map<String, Map<String, String>>>(); - Map<String, Map<String, String>> configTags = - managementController.findConfigurationTagsWithOverrides(cluster, hostName); + Map<String, Map<String, String>> configTags = new TreeMap<String, Map<String, String>>(); + + ExecutionCommand execCmd = stage.getExecutionCommandWrapper(hostName, + componentName).getExecutionCommand(); + + // if the command should fetch brand new configuration tags before + // execution, then we don't need to fetch them now + if(actionExecutionContext.getParameters() != null && actionExecutionContext.getParameters().containsKey(KeyNames.REFRESH_CONFIG_TAGS_BEFORE_EXECUTION)){ + execCmd.setForceRefreshConfigTagsBeforeExecution(true); + } + + // when building complex orchestration ahead of time (such as when + // performing ugprades), fetching configuration tags can take a very long + // time - if it's not needed, then don't do it + if (!execCmd.getForceRefreshConfigTagsBeforeExecution()) { + configTags = managementController.findConfigurationTagsWithOverrides(cluster, hostName); + } HostRoleCommand cmd = stage.getHostRoleCommand(hostName, componentName); if (cmd != null) { @@ -347,9 +363,6 @@ public class AmbariCustomCommandExecutionHelper { cmd.setCustomCommandName(commandName); } - ExecutionCommand execCmd = stage.getExecutionCommandWrapper(hostName, - componentName).getExecutionCommand(); - //set type background if(customCommandDefinition != null && customCommandDefinition.isBackground()){ execCmd.setCommandType(AgentCommandType.BACKGROUND_EXECUTION_COMMAND); @@ -359,14 +372,6 @@ public class AmbariCustomCommandExecutionHelper { execCmd.setConfigurationAttributes(configurationAttributes); execCmd.setConfigurationTags(configTags); - if(actionExecutionContext.getParameters() != null && actionExecutionContext.getParameters().containsKey(KeyNames.REFRESH_ADITIONAL_COMPONENT_TAGS)){ - execCmd.setForceRefreshConfigTags(parseAndValidateComponentsMapping(actionExecutionContext.getParameters().get(KeyNames.REFRESH_ADITIONAL_COMPONENT_TAGS))); - } - - if(actionExecutionContext.getParameters() != null && actionExecutionContext.getParameters().containsKey(KeyNames.REFRESH_CONFIG_TAGS_BEFORE_EXECUTION)){ - execCmd.setForceRefreshConfigTagsBeforeExecution(parseAndValidateComponentsMapping(actionExecutionContext.getParameters().get(KeyNames.REFRESH_CONFIG_TAGS_BEFORE_EXECUTION))); - } - execCmd.setAvailableServicesFromServiceInfoMap(ambariMetaInfo.getServices(stackId.getStackName(), stackId.getStackVersion())); Map<String, String> hostLevelParams = new TreeMap<String, String>(); @@ -378,15 +383,17 @@ public class AmbariCustomCommandExecutionHelper { hostLevelParams.put(STACK_NAME, stackId.getStackName()); hostLevelParams.put(STACK_VERSION, stackId.getStackVersion()); - Set<String> userSet = configHelper.getPropertyValuesWithPropertyType(stackId, PropertyType.USER, cluster); + Map<String, DesiredConfig> desiredConfigs = cluster.getDesiredConfigs(); + + Set<String> userSet = configHelper.getPropertyValuesWithPropertyType(stackId, PropertyType.USER, cluster, desiredConfigs); String userList = gson.toJson(userSet); hostLevelParams.put(USER_LIST, userList); - Set<String> groupSet = configHelper.getPropertyValuesWithPropertyType(stackId, PropertyType.GROUP, cluster); + Set<String> groupSet = configHelper.getPropertyValuesWithPropertyType(stackId, PropertyType.GROUP, cluster, desiredConfigs); String groupList = gson.toJson(groupSet); hostLevelParams.put(GROUP_LIST, groupList); - Set<String> notManagedHdfsPathSet = configHelper.getPropertyValuesWithPropertyType(stackId, PropertyType.NOT_MANAGED_HDFS_PATH, cluster); + Set<String> notManagedHdfsPathSet = configHelper.getPropertyValuesWithPropertyType(stackId, PropertyType.NOT_MANAGED_HDFS_PATH, cluster, desiredConfigs); String notManagedHdfsPathList = gson.toJson(notManagedHdfsPathSet); hostLevelParams.put(NOT_MANAGED_HDFS_PATH_LIST, notManagedHdfsPathList); @@ -458,8 +465,17 @@ public class AmbariCustomCommandExecutionHelper { private void applyCustomCommandBackendLogic(Cluster cluster, String serviceName, String componentName, String commandName, String hostname) throws AmbariException { switch (commandName) { case "RESTART": - LOG.info("Updating desired state on RESTART for the service [{}], service component [{}]", serviceName, componentName); - cluster.getService(serviceName).getServiceComponent(componentName).getServiceComponentHost(hostname).setDesiredState(State.STARTED); + ServiceComponentHost serviceComponentHost = cluster.getService( + serviceName).getServiceComponent(componentName).getServiceComponentHost(hostname); + + State currentDesiredState = serviceComponentHost.getDesiredState(); + if (currentDesiredState != State.STARTED) { + LOG.info("Updating desired state to {} on RESTART for {}/{} because it was {}", + State.STARTED, serviceName, componentName, currentDesiredState); + + serviceComponentHost.setDesiredState(State.STARTED); + } + break; default: LOG.debug("No backend operations needed for the custom command: {}", commandName); @@ -467,21 +483,6 @@ public class AmbariCustomCommandExecutionHelper { } } - /** - * Splits the passed comma separated value and returns it as set. - * - * @param commaSeparatedTags separated list - * - * @return set of items or null - */ - private Set<String> parseAndValidateComponentsMapping(String commaSeparatedTags) { - Set<String> retVal = null; - if(commaSeparatedTags != null && !commaSeparatedTags.trim().isEmpty()){ - Collections.addAll(retVal = new HashSet<String>(), commaSeparatedTags.split(",")); - } - return retVal; - } - private void findHostAndAddServiceCheckAction(final ActionExecutionContext actionExecutionContext, final RequestResourceFilter resourceFilter, Stage stage) throws AmbariException { @@ -606,18 +607,27 @@ public class AmbariCustomCommandExecutionHelper { new TreeMap<String, Map<String, String>>(); Map<String, Map<String, Map<String, String>>> configurationAttributes = new TreeMap<String, Map<String, Map<String, String>>>(); - Map<String, Map<String, String>> configTags = - managementController.findConfigurationTagsWithOverrides(cluster, hostname); + Map<String, Map<String, String>> configTags = new TreeMap<String, Map<String, String>>(); ExecutionCommand execCmd = stage.getExecutionCommandWrapper(hostname, smokeTestRole).getExecutionCommand(); + // if the command should fetch brand new configuration tags before + // execution, then we don't need to fetch them now + if(actionParameters != null && actionParameters.containsKey(KeyNames.REFRESH_CONFIG_TAGS_BEFORE_EXECUTION)){ + execCmd.setForceRefreshConfigTagsBeforeExecution(true); + } + + // when building complex orchestration ahead of time (such as when + // performing ugprades), fetching configuration tags can take a very long + // time - if it's not needed, then don't do it + if (!execCmd.getForceRefreshConfigTagsBeforeExecution()) { + configTags = managementController.findConfigurationTagsWithOverrides(cluster, hostname); + } + execCmd.setConfigurations(configurations); execCmd.setConfigurationAttributes(configurationAttributes); execCmd.setConfigurationTags(configTags); - if(actionParameters != null && actionParameters.containsKey(KeyNames.REFRESH_CONFIG_TAGS_BEFORE_EXECUTION)){ - execCmd.setForceRefreshConfigTagsBeforeExecution(parseAndValidateComponentsMapping(actionParameters.get(KeyNames.REFRESH_CONFIG_TAGS_BEFORE_EXECUTION))); - } // Generate cluster host info execCmd.setClusterHostInfo( @@ -631,15 +641,15 @@ public class AmbariCustomCommandExecutionHelper { Map<String, String> commandParams = new TreeMap<String, String>(); //Propagate HCFS service type info - Iterator<Service> it = cluster.getServices().values().iterator(); - while(it.hasNext()) { - ServiceInfo serviceInfoInstance = ambariMetaInfo.getService(stackId.getStackName(),stackId.getStackVersion(), it.next().getName()); - LOG.info("Iterating service type Instance in addServiceCheckAction:: " + serviceInfoInstance.getName()); - if(serviceInfoInstance.getServiceType() != null) { - LOG.info("Adding service type info in addServiceCheckAction:: " + serviceInfoInstance.getServiceType()); - commandParams.put("dfs_type",serviceInfoInstance.getServiceType()); - break; - } + Map<String, ServiceInfo> serviceInfos = ambariMetaInfo.getServices(stackId.getStackName(), stackId.getStackVersion()); + for (ServiceInfo serviceInfoInstance : serviceInfos.values()) { + if (serviceInfoInstance.getServiceType() != null) { + LOG.debug("Adding {} to command parameters for {}", serviceInfoInstance.getServiceType(), + serviceInfoInstance.getName()); + + commandParams.put("dfs_type", serviceInfoInstance.getServiceType()); + break; + } } String commandTimeout = configs.getDefaultAgentTaskTimeout(false); @@ -993,10 +1003,6 @@ public class AmbariCustomCommandExecutionHelper { extraParams.put(componentName, requestParams.get(componentName)); } - if(requestParams.containsKey(KeyNames.REFRESH_ADITIONAL_COMPONENT_TAGS)){ - actionExecutionContext.getParameters().put(KeyNames.REFRESH_ADITIONAL_COMPONENT_TAGS, requestParams.get(KeyNames.REFRESH_ADITIONAL_COMPONENT_TAGS)); - } - // If command should be retried upon failure then add the option and also the default duration for retry if (requestParams.containsKey(KeyNames.COMMAND_RETRY_ENABLED)) { extraParams.put(KeyNames.COMMAND_RETRY_ENABLED, requestParams.get(KeyNames.COMMAND_RETRY_ENABLED)); @@ -1186,17 +1192,16 @@ public class AmbariCustomCommandExecutionHelper { clusterHostInfoJson = StageUtils.getGson().toJson(clusterHostInfo); //Propogate HCFS service type info to command params - Iterator<Service> it = cluster.getServices().values().iterator(); - while(it.hasNext()) { - ServiceInfo serviceInfoInstance = ambariMetaInfo.getService(stackId.getStackName(),stackId.getStackVersion(), it.next().getName()); - LOG.info("Iterating service type Instance in getCommandJson:: " + serviceInfoInstance.getName()); - if(serviceInfoInstance.getServiceType() != null) { - LOG.info("Adding service type info in getCommandJson:: " + serviceInfoInstance.getServiceType()); - commandParamsStage.put("dfs_type",serviceInfoInstance.getServiceType()); - break; - } + Map<String, ServiceInfo> serviceInfos = ambariMetaInfo.getServices(stackId.getStackName(), stackId.getStackVersion()); + for (ServiceInfo serviceInfoInstance : serviceInfos.values()) { + if (serviceInfoInstance.getServiceType() != null) { + LOG.debug("Adding {} to command parameters for {}", serviceInfoInstance.getServiceType(), + serviceInfoInstance.getName()); + + commandParamsStage.put("dfs_type", serviceInfoInstance.getServiceType()); + break; + } } - } String hostParamsStageJson = StageUtils.getGson().toJson(hostParamsStage); @@ -1229,7 +1234,8 @@ public class AmbariCustomCommandExecutionHelper { hostLevelParams.put(AGENT_STACK_RETRY_ON_UNAVAILABILITY, configs.isAgentStackRetryOnInstallEnabled()); hostLevelParams.put(AGENT_STACK_RETRY_COUNT, configs.getAgentStackRetryOnInstallCount()); - Set<String> notManagedHdfsPathSet = configHelper.getPropertyValuesWithPropertyType(stackId, PropertyType.NOT_MANAGED_HDFS_PATH, cluster); + Map<String, DesiredConfig> desiredConfigs = cluster.getDesiredConfigs(); + Set<String> notManagedHdfsPathSet = configHelper.getPropertyValuesWithPropertyType(stackId, PropertyType.NOT_MANAGED_HDFS_PATH, cluster, desiredConfigs); String notManagedHdfsPathList = gson.toJson(notManagedHdfsPathSet); hostLevelParams.put(NOT_MANAGED_HDFS_PATH_LIST, notManagedHdfsPathList); http://git-wip-us.apache.org/repos/asf/ambari/blob/3ac13b5c/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java index 066acab..aba45bf 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java @@ -2281,15 +2281,17 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle String packageList = gson.toJson(packages); hostParams.put(PACKAGE_LIST, packageList); - Set<String> userSet = configHelper.getPropertyValuesWithPropertyType(stackId, PropertyType.USER, cluster); + Map<String, DesiredConfig> desiredConfigs = cluster.getDesiredConfigs(); + + Set<String> userSet = configHelper.getPropertyValuesWithPropertyType(stackId, PropertyType.USER, cluster, desiredConfigs); String userList = gson.toJson(userSet); hostParams.put(USER_LIST, userList); - Set<String> groupSet = configHelper.getPropertyValuesWithPropertyType(stackId, PropertyType.GROUP, cluster); + Set<String> groupSet = configHelper.getPropertyValuesWithPropertyType(stackId, PropertyType.GROUP, cluster, desiredConfigs); String groupList = gson.toJson(groupSet); hostParams.put(GROUP_LIST, groupList); - Set<String> notManagedHdfsPathSet = configHelper.getPropertyValuesWithPropertyType(stackId, PropertyType.NOT_MANAGED_HDFS_PATH, cluster); + Set<String> notManagedHdfsPathSet = configHelper.getPropertyValuesWithPropertyType(stackId, PropertyType.NOT_MANAGED_HDFS_PATH, cluster, desiredConfigs); String notManagedHdfsPathList = gson.toJson(notManagedHdfsPathSet); hostParams.put(NOT_MANAGED_HDFS_PATH_LIST, notManagedHdfsPathList); http://git-wip-us.apache.org/repos/asf/ambari/blob/3ac13b5c/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProvider.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProvider.java index af24a69..c390c86 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProvider.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProvider.java @@ -17,10 +17,44 @@ */ package org.apache.ambari.server.controller.internal; -import com.google.gson.Gson; -import com.google.inject.assistedinject.Assisted; -import com.google.inject.assistedinject.AssistedInject; -import com.google.inject.persist.Transactional; +import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.AGENT_STACK_RETRY_COUNT; +import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.AGENT_STACK_RETRY_ON_UNAVAILABILITY; +import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.DB_NAME; +import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.GROUP_LIST; +import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.HOST_SYS_PREPPED; +import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.JAVA_HOME; +import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.JAVA_VERSION; +import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.JCE_NAME; +import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.JDK_LOCATION; +import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.JDK_NAME; +import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.MYSQL_JDBC_URL; +import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.NOT_MANAGED_HDFS_PATH_LIST; +import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.ORACLE_JDBC_URL; +import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.PACKAGE_LIST; +import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.SERVICE_REPO_INFO; +import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.STACK_NAME; +import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.STACK_VERSION; +import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.USER_LIST; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.PrintWriter; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeoutException; + import org.apache.ambari.server.AmbariException; import org.apache.ambari.server.api.services.AmbariMetaInfo; import org.apache.ambari.server.configuration.Configuration; @@ -54,43 +88,10 @@ import org.apache.ambari.server.utils.StageUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.BufferedReader; -import java.io.File; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.io.PrintWriter; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.TreeMap; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.TimeoutException; - -import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.AGENT_STACK_RETRY_COUNT; -import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.AGENT_STACK_RETRY_ON_UNAVAILABILITY; -import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.DB_NAME; -import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.GROUP_LIST; -import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.JAVA_HOME; -import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.JAVA_VERSION; -import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.JCE_NAME; -import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.JDK_LOCATION; -import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.JDK_NAME; -import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.MYSQL_JDBC_URL; -import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.NOT_MANAGED_HDFS_PATH_LIST; -import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.ORACLE_JDBC_URL; -import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.PACKAGE_LIST; -import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.SERVICE_REPO_INFO; -import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.STACK_NAME; -import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.STACK_VERSION; -import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.USER_LIST; -import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.HOST_SYS_PREPPED; +import com.google.gson.Gson; +import com.google.inject.assistedinject.Assisted; +import com.google.inject.assistedinject.AssistedInject; +import com.google.inject.persist.Transactional; /** * Resource provider for client config resources. @@ -194,7 +195,7 @@ public class ClientConfigResourceProvider extends AbstractControllerResourceProv getComponent(stackId.getStackName(), stackId.getStackVersion(), serviceName, componentName); packageFolder = managementController.getAmbariMetaInfo(). getService(stackId.getStackName(), stackId.getStackVersion(), serviceName).getServicePackageFolder(); - + String commandScript = componentInfo.getCommandScript().getScript(); List<ClientConfigFileDefinition> clientConfigFiles = componentInfo.getClientConfigFiles(); @@ -204,7 +205,7 @@ public class ClientConfigResourceProvider extends AbstractControllerResourceProv String resourceDirPath = configs.getResourceDirPath(); String packageFolderAbsolute = resourceDirPath + File.separator + packageFolder; - + String commandScriptAbsolute = packageFolderAbsolute + File.separator + commandScript; @@ -336,15 +337,15 @@ public class ClientConfigResourceProvider extends AbstractControllerResourceProv String packageList = gson.toJson(packages); hostLevelParams.put(PACKAGE_LIST, packageList); - Set<String> userSet = configHelper.getPropertyValuesWithPropertyType(stackId, PropertyType.USER, cluster); + Set<String> userSet = configHelper.getPropertyValuesWithPropertyType(stackId, PropertyType.USER, cluster, desiredClusterConfigs); String userList = gson.toJson(userSet); hostLevelParams.put(USER_LIST, userList); - Set<String> groupSet = configHelper.getPropertyValuesWithPropertyType(stackId, PropertyType.GROUP, cluster); + Set<String> groupSet = configHelper.getPropertyValuesWithPropertyType(stackId, PropertyType.GROUP, cluster, desiredClusterConfigs); String groupList = gson.toJson(groupSet); hostLevelParams.put(GROUP_LIST, groupList); - Set<String> notManagedHdfsPathSet = configHelper.getPropertyValuesWithPropertyType(stackId, PropertyType.NOT_MANAGED_HDFS_PATH, cluster); + Set<String> notManagedHdfsPathSet = configHelper.getPropertyValuesWithPropertyType(stackId,PropertyType.NOT_MANAGED_HDFS_PATH, cluster, desiredClusterConfigs); String notManagedHdfsPathList = gson.toJson(notManagedHdfsPathSet); hostLevelParams.put(NOT_MANAGED_HDFS_PATH_LIST, notManagedHdfsPathList); @@ -484,13 +485,14 @@ public class ClientConfigResourceProvider extends AbstractControllerResourceProv commandLineThread.join(timeout); logStreamThread.join(timeout); Integer returnCode = commandLineThread.getReturnCode(); - if (returnCode == null) + if (returnCode == null) { throw new TimeoutException(); - else if (returnCode != 0) + } else if (returnCode != 0) { throw new ExecutionException(String.format("Execution of \"%s\" returned %d.", commandLine, returnCode), new Throwable(logStream.getOutput())); - else + } else { return commandLineThread.returnCode; + } } catch (InterruptedException ex) { commandLineThread.interrupt(); Thread.currentThread().interrupt(); @@ -505,7 +507,7 @@ public class ClientConfigResourceProvider extends AbstractControllerResourceProv private Integer returnCode; public void setReturnCode(Integer exit) { - this.returnCode = exit; + returnCode = exit; } public Integer getReturnCode() { @@ -517,6 +519,7 @@ public class ClientConfigResourceProvider extends AbstractControllerResourceProv } + @Override public void run() { try { setReturnCode(process.waitFor()); @@ -533,14 +536,15 @@ public class ClientConfigResourceProvider extends AbstractControllerResourceProv private StringBuilder output; public LogStreamReader(InputStream is) { - this.reader = new BufferedReader(new InputStreamReader(is)); - this.output = new StringBuilder(""); + reader = new BufferedReader(new InputStreamReader(is)); + output = new StringBuilder(""); } public String getOutput() { - return this.output.toString(); + return output.toString(); } + @Override public void run() { try { String line = reader.readLine(); http://git-wip-us.apache.org/repos/asf/ambari/blob/3ac13b5c/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java index c390f44..55e9dbd 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java @@ -1703,7 +1703,7 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider */ private Map<String, String> getNewParameterMap(RequestStageContainer requestStageContainer) { Map<String, String> parameters = new HashMap<String, String>(); - parameters.put(KeyNames.REFRESH_CONFIG_TAGS_BEFORE_EXECUTION, "*"); + parameters.put(KeyNames.REFRESH_CONFIG_TAGS_BEFORE_EXECUTION, "true"); parameters.put(COMMAND_PARAM_REQUEST_ID, String.valueOf(requestStageContainer.getId())); return parameters; } http://git-wip-us.apache.org/repos/asf/ambari/blob/3ac13b5c/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java index 216d850..16edc74 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java @@ -35,8 +35,6 @@ import java.util.concurrent.ThreadFactory; import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.ReentrantReadWriteLock; -import org.apache.ambari.annotations.TransactionalLock; -import org.apache.ambari.annotations.TransactionalLock.LockArea; import org.apache.ambari.server.AmbariException; import org.apache.ambari.server.api.services.AmbariMetaInfo; import org.apache.ambari.server.configuration.Configuration; @@ -590,9 +588,9 @@ public class ConfigHelper { return result; } - public Set<String> getPropertyValuesWithPropertyType(StackId stackId, PropertyType propertyType, Cluster cluster) throws AmbariException { + public Set<String> getPropertyValuesWithPropertyType(StackId stackId, PropertyType propertyType, + Cluster cluster, Map<String, DesiredConfig> desiredConfigs) throws AmbariException { StackInfo stack = ambariMetaInfo.getStack(stackId.getStackName(), stackId.getStackVersion()); - Map<String, DesiredConfig> desiredConfigs = cluster.getDesiredConfigs(); Map<String, Config> actualConfigs = new HashMap<>(); Set<String> result = new HashSet<String>(); @@ -1417,7 +1415,7 @@ public class ConfigHelper { public void run() { staleConfigCacheLock.writeLock().lock(); try { - ConfigHelper.this.staleConfigCacheDesiredConfigs = m_desiredConfigs; + staleConfigCacheDesiredConfigs = m_desiredConfigs; if (null == m_keysToInvalidate || m_keysToInvalidate.isEmpty()) { staleConfigsCache.invalidateAll(); } else { http://git-wip-us.apache.org/repos/asf/ambari/blob/3ac13b5c/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeHelper.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeHelper.java index 8dcd163..6cefa79 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeHelper.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeHelper.java @@ -71,6 +71,7 @@ import org.slf4j.LoggerFactory; import com.google.inject.Inject; import com.google.inject.Provider; import com.google.inject.Singleton; +import com.google.inject.persist.Transactional; /** * Class to assist with upgrading a cluster. @@ -749,12 +750,21 @@ public class UpgradeHelper { } /** - * Transitions all affected components to upgrading state. Transition is performed - * only for components that advertise their version. Service component desired - * version is set to one passed as an argument - * @param version desired version (like 2.2.1.0-1234) for upgrade - * @param targetServices targets for upgrade + * Transitions all affected components to {@link UpgradeState#IN_PROGRESS}. + * Transition is performed only for components that advertise their version. + * Additionally sets the service component desired version to the specified + * argument. + * <p/> + * Because this iterates over all of the components on every host and updates + * the upgrade state individually, we wrap this method inside of a transaction + * to prevent 1000's of transactions from being opened and committed. + * + * @param version + * desired version (like 2.2.1.0-1234) for upgrade + * @param targetServices + * targets for upgrade */ + @Transactional public void putComponentsToUpgradingState(String version, Map<Service, Set<ServiceComponent>> targetServices) throws AmbariException { // TODO: generalize method? http://git-wip-us.apache.org/repos/asf/ambari/blob/3ac13b5c/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelperTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelperTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelperTest.java index 7a5f377..e82900d 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelperTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelperTest.java @@ -148,7 +148,7 @@ public class AmbariCustomCommandExecutionHelperTest { ExecuteActionRequest actionRequest = new ExecuteActionRequest("c1", "REFRESHQUEUES", new HashMap<String, String>() { { - put("forceRefreshConfigTags", "capacity-scheduler"); + put("forceRefreshConfigTagsBeforeExecution", "true"); } }, false); actionRequest.getResourceFilters().add(new RequestResourceFilter("YARN", "RESOURCEMANAGER", Collections.singletonList("c1-c6401"))); @@ -169,11 +169,7 @@ public class AmbariCustomCommandExecutionHelperTest { Assert.assertEquals(1, commands.size()); ExecutionCommand command = commands.get(0).getExecutionCommand(); - - Assert.assertNotNull(command.getForceRefreshConfigTags()); - Assert.assertEquals(1, command.getForceRefreshConfigTags().size()); - Assert.assertEquals("capacity-scheduler", - command.getForceRefreshConfigTags().iterator().next()); + Assert.assertEquals(true, command.getForceRefreshConfigTagsBeforeExecution()); } @Test http://git-wip-us.apache.org/repos/asf/ambari/blob/3ac13b5c/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerImplTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerImplTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerImplTest.java index 37a2e63..e54a117 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerImplTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerImplTest.java @@ -18,8 +18,6 @@ package org.apache.ambari.server.controller; -import javax.persistence.RollbackException; -import junit.framework.Assert; import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.DB_DRIVER_FILENAME; import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.HOST_SYS_PREPPED; import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.JAVA_VERSION; @@ -59,6 +57,8 @@ import java.util.Map; import java.util.Properties; import java.util.Set; +import javax.persistence.RollbackException; + import org.apache.ambari.server.AmbariException; import org.apache.ambari.server.ClusterNotFoundException; import org.apache.ambari.server.HostNotFoundException; @@ -89,6 +89,7 @@ import org.apache.ambari.server.state.Clusters; import org.apache.ambari.server.state.ComponentInfo; import org.apache.ambari.server.state.ConfigHelper; import org.apache.ambari.server.state.ConfigImpl; +import org.apache.ambari.server.state.DesiredConfig; import org.apache.ambari.server.state.Host; import org.apache.ambari.server.state.MaintenanceState; import org.apache.ambari.server.state.PropertyInfo; @@ -119,6 +120,8 @@ import com.google.inject.Injector; import com.google.inject.Module; import com.google.inject.util.Modules; +import junit.framework.Assert; + /** * AmbariManagementControllerImpl unit tests */ @@ -1992,8 +1995,11 @@ public class AmbariManagementControllerImplTest { RepositoryVersionEntity repositoryVersionEntity = createNiceMock(RepositoryVersionEntity.class); ConfigHelper configHelper = createNiceMock(ConfigHelper.class); + Map<String, DesiredConfig> desiredConfigs = new HashMap<>(); + expect(cluster.getClusterName()).andReturn(clusterName); expect(cluster.getDesiredStackVersion()).andReturn(stackId); + expect(cluster.getDesiredConfigs()).andReturn(desiredConfigs); expect(stackId.getStackName()).andReturn(SOME_STACK_NAME).anyTimes(); expect(stackId.getStackVersion()).andReturn(SOME_STACK_VERSION).anyTimes(); expect(configuration.getMySQLJarName()).andReturn(MYSQL_JAR); @@ -2009,8 +2015,9 @@ public class AmbariManagementControllerImplTest { expect(clusterVersionDAO.findByClusterAndStateCurrent(clusterName)).andReturn(clusterVersionEntity).anyTimes(); expect(clusterVersionEntity.getRepositoryVersion()).andReturn(repositoryVersionEntity).anyTimes(); expect(repositoryVersionEntity.getVersion()).andReturn("1234").anyTimes(); - expect(configHelper.getPropertyValuesWithPropertyType(stackId, PropertyInfo.PropertyType.NOT_MANAGED_HDFS_PATH, - cluster)).andReturn(notManagedHdfsPathSet); + expect(configHelper.getPropertyValuesWithPropertyType(stackId, + PropertyInfo.PropertyType.NOT_MANAGED_HDFS_PATH, cluster, desiredConfigs)).andReturn( + notManagedHdfsPathSet); replay(manager, clusters, cluster, injector, stackId, configuration, clusterVersionDAO, clusterVersionEntity, repositoryVersionEntity, configHelper); http://git-wip-us.apache.org/repos/asf/ambari/blob/3ac13b5c/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProviderTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProviderTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProviderTest.java index 9603226..f0bddf8 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProviderTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProviderTest.java @@ -18,15 +18,60 @@ package org.apache.ambari.server.controller.internal; +import static org.easymock.EasyMock.anyObject; +import static org.easymock.EasyMock.createMock; +import static org.easymock.EasyMock.createNiceMock; +import static org.easymock.EasyMock.expect; +import static org.easymock.EasyMock.replay; +import static org.easymock.EasyMock.verify; +import static org.junit.Assert.assertFalse; + +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.InputStream; +import java.io.PrintWriter; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Set; + import org.apache.ambari.server.api.services.AmbariMetaInfo; import org.apache.ambari.server.configuration.Configuration; -import org.apache.ambari.server.controller.*; -import org.apache.ambari.server.controller.spi.*; +import org.apache.ambari.server.controller.AmbariManagementController; +import org.apache.ambari.server.controller.RequestStatusResponse; +import org.apache.ambari.server.controller.ServiceComponentHostRequest; +import org.apache.ambari.server.controller.ServiceComponentHostResponse; +import org.apache.ambari.server.controller.spi.Predicate; +import org.apache.ambari.server.controller.spi.Request; +import org.apache.ambari.server.controller.spi.Resource; +import org.apache.ambari.server.controller.spi.ResourceProvider; +import org.apache.ambari.server.controller.spi.SystemException; import org.apache.ambari.server.controller.utilities.PredicateBuilder; import org.apache.ambari.server.controller.utilities.PropertyHelper; -import org.apache.ambari.server.state.*; +import org.apache.ambari.server.stack.StackManager; +import org.apache.ambari.server.state.ClientConfigFileDefinition; +import org.apache.ambari.server.state.Cluster; +import org.apache.ambari.server.state.Clusters; +import org.apache.ambari.server.state.CommandScriptDefinition; +import org.apache.ambari.server.state.ComponentInfo; +import org.apache.ambari.server.state.Config; +import org.apache.ambari.server.state.ConfigHelper; +import org.apache.ambari.server.state.DesiredConfig; +import org.apache.ambari.server.state.Host; import org.apache.ambari.server.state.PropertyInfo; +import org.apache.ambari.server.state.Service; +import org.apache.ambari.server.state.ServiceComponent; +import org.apache.ambari.server.state.ServiceComponentHost; +import org.apache.ambari.server.state.ServiceInfo; +import org.apache.ambari.server.state.ServiceOsSpecific; +import org.apache.ambari.server.state.StackId; import org.apache.ambari.server.utils.StageUtils; +import org.easymock.EasyMock; import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; @@ -34,17 +79,6 @@ import org.powermock.api.easymock.PowerMock; import org.powermock.core.classloader.annotations.PrepareForTest; import org.powermock.modules.junit4.PowerMockRunner; -import java.io.ByteArrayInputStream; -import java.io.File; -import java.io.InputStream; -import java.io.PrintWriter; -import java.util.*; - -import org.apache.ambari.server.stack.StackManager; - -import static org.easymock.EasyMock.*; -import static org.junit.Assert.assertFalse; - /** * TaskResourceProvider tests. */ @@ -224,7 +258,7 @@ public class ClientConfigResourceProviderTest { Map<String, String> returnConfigMap = new HashMap<String, String>(); returnConfigMap.put(Configuration.SERVER_TMP_DIR_KEY, Configuration.SERVER_TMP_DIR_DEFAULT); returnConfigMap.put(Configuration.AMBARI_PYTHON_WRAP_KEY, Configuration.AMBARI_PYTHON_WRAP_DEFAULT); - + // set expectations expect(managementController.getConfigHelper()).andReturn(configHelper); expect(managementController.getAmbariMetaInfo()).andReturn(ambariMetaInfo).anyTimes(); @@ -291,7 +325,7 @@ public class ClientConfigResourceProviderTest { expect(serviceInfo.getOsSpecifics()).andReturn(new HashMap<String, ServiceOsSpecific>()).anyTimes(); Set<String> userSet = new HashSet<String>(); userSet.add("hdfs"); - expect(configHelper.getPropertyValuesWithPropertyType(stackId, PropertyInfo.PropertyType.USER, cluster)).andReturn(userSet); + expect(configHelper.getPropertyValuesWithPropertyType(stackId, PropertyInfo.PropertyType.USER, cluster, desiredConfigMap)).andReturn(userSet); PowerMock.expectNew(File.class, new Class<?>[]{String.class}, anyObject(String.class)).andReturn(mockFile).anyTimes(); PowerMock.createNiceMockAndExpectNew(PrintWriter.class, anyObject()); expect(mockFile.getParent()).andReturn(""); @@ -422,11 +456,11 @@ public class ClientConfigResourceProviderTest { Set<ServiceComponentHostResponse> responses = new LinkedHashSet<ServiceComponentHostResponse>(); responses.add(shr1); - + Map<String, String> returnConfigMap = new HashMap<String, String>(); returnConfigMap.put(Configuration.SERVER_TMP_DIR_KEY, Configuration.SERVER_TMP_DIR_DEFAULT); returnConfigMap.put(Configuration.AMBARI_PYTHON_WRAP_KEY, Configuration.AMBARI_PYTHON_WRAP_DEFAULT); - + // set expectations expect(managementController.getConfigHelper()).andReturn(configHelper); expect(managementController.getAmbariMetaInfo()).andReturn(ambariMetaInfo).anyTimes(); @@ -494,7 +528,7 @@ public class ClientConfigResourceProviderTest { expect(serviceInfo.getOsSpecifics()).andReturn(new HashMap<String, ServiceOsSpecific>()).anyTimes(); Set<String> userSet = new HashSet<String>(); userSet.add("hdfs"); - expect(configHelper.getPropertyValuesWithPropertyType(stackId, PropertyInfo.PropertyType.USER, cluster)).andReturn(userSet); + expect(configHelper.getPropertyValuesWithPropertyType(stackId, PropertyInfo.PropertyType.USER, cluster, desiredConfigMap)).andReturn(userSet); PowerMock.expectNew(File.class, new Class<?>[]{String.class}, anyObject(String.class)).andReturn(mockFile).anyTimes(); PowerMock.createNiceMockAndExpectNew(PrintWriter.class, anyObject()); expect(mockFile.getParent()).andReturn(""); @@ -532,9 +566,9 @@ public class ClientConfigResourceProviderTest { runtime, process); PowerMock.verifyAll(); } - - - + + + @Test public void testDeleteResources() throws Exception { Resource.Type type = Resource.Type.ClientConfig; http://git-wip-us.apache.org/repos/asf/ambari/blob/3ac13b5c/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/UpgradeResourceProviderHDP22Test.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/UpgradeResourceProviderHDP22Test.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/UpgradeResourceProviderHDP22Test.java index 115b518..139702c 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/UpgradeResourceProviderHDP22Test.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/UpgradeResourceProviderHDP22Test.java @@ -18,9 +18,9 @@ package org.apache.ambari.server.controller.internal; import static org.easymock.EasyMock.anyLong; +import static org.easymock.EasyMock.createNiceMock; import static org.easymock.EasyMock.eq; import static org.easymock.EasyMock.expect; -import static org.easymock.EasyMock.createNiceMock; import static org.easymock.EasyMock.replay; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -66,7 +66,6 @@ import org.apache.ambari.server.security.authorization.RoleAuthorization; import org.apache.ambari.server.state.Cluster; import org.apache.ambari.server.state.Clusters; import org.apache.ambari.server.state.Config; -import org.apache.ambari.server.state.ConfigHelper; import org.apache.ambari.server.state.ConfigImpl; import org.apache.ambari.server.state.Host; import org.apache.ambari.server.state.HostState; @@ -78,24 +77,21 @@ import org.apache.ambari.server.state.StackId; import org.apache.ambari.server.topology.TopologyManager; import org.apache.ambari.server.utils.StageUtils; import org.apache.ambari.server.view.ViewRegistry; -import org.easymock.EasyMock; import org.junit.After; import org.junit.Before; import org.junit.Test; - -import com.google.gson.Gson; -import com.google.inject.Binder; -import com.google.inject.Guice; -import com.google.inject.Injector; -import com.google.inject.Module; -import com.google.inject.persist.PersistService; -import com.google.inject.util.Modules; import org.junit.runner.RunWith; import org.powermock.api.easymock.PowerMock; import org.powermock.core.classloader.annotations.PowerMockIgnore; import org.powermock.core.classloader.annotations.PrepareForTest; import org.powermock.modules.junit4.PowerMockRunner; +import com.google.common.collect.ImmutableMap; +import com.google.gson.Gson; +import com.google.inject.Guice; +import com.google.inject.Injector; +import com.google.inject.persist.PersistService; + /** * UpgradeResourceDefinition tests. */ @@ -110,81 +106,22 @@ public class UpgradeResourceProviderHDP22Test { private Clusters clusters; private OrmTestHelper helper; private AmbariManagementController amc; - private ConfigHelper configHelper; private StackDAO stackDAO; private TopologyManager topologyManager; private static final String configTagVersion1 = "version1"; private static final String configTagVersion2 = "version2"; - @SuppressWarnings("serial") - private static final Map<String, String> configTagVersion1Properties = new HashMap<String, String>() { - { - put("hive.server2.thrift.port", "10000"); - } - }; - private static final Map<String, String> configTagVersion2Properties = new HashMap<String, String>() { - { - put("hive.server2.thrift.port", "10010"); - } - }; + private static final Map<String, String> configTagVersion1Properties = new ImmutableMap.Builder<String, String>().put( + "hive.server2.thrift.port", "10000").build(); + + private static final Map<String, String> configTagVersion2Properties = new ImmutableMap.Builder<String, String>().put( + "hive.server2.thrift.port", "10010").build(); - @SuppressWarnings({ "serial", "unchecked" }) @Before public void before() throws Exception { - // setup the config helper for placeholder resolution - configHelper = EasyMock.createNiceMock(ConfigHelper.class); - - expect(configHelper.getPlaceholderValueFromDesiredConfigurations(EasyMock.anyObject(Cluster.class), EasyMock.eq("{{foo/bar}}"))).andReturn( - "placeholder-rendered-properly").anyTimes(); - - expect(configHelper.getDefaultProperties(EasyMock.anyObject(StackId.class), EasyMock.anyObject(Cluster.class))).andReturn( - new HashMap<String, Map<String, String>>()).anyTimes(); - - expect(configHelper.getEffectiveConfigAttributes(EasyMock.anyObject(Cluster.class), EasyMock.anyObject(Map.class))).andReturn( - new HashMap<String, Map<String, Map<String, String>>>()).anyTimes(); - - expect(configHelper.getEffectiveDesiredTags(EasyMock.anyObject(Cluster.class), EasyMock.eq("h1"))).andReturn(new HashMap<String, Map<String, String>>() { - { - put("hive-site", new HashMap<String, String>() { - { - put("tag", configTagVersion1); - } - }); - } - }).times(3); - - expect(configHelper.getEffectiveDesiredTags(EasyMock.anyObject(Cluster.class), EasyMock.eq("h1"))).andReturn(new HashMap<String, Map<String, String>>() { - { - put("hive-site", new HashMap<String, String>() { - { - put("tag", configTagVersion2); - } - }); - } - }).times(2); - - expect(configHelper.getEffectiveConfigProperties(EasyMock.anyObject(Cluster.class), EasyMock.anyObject(Map.class))).andReturn( - new HashMap<String, Map<String, String>>() { - { - put("hive-site", configTagVersion1Properties); - } - }).times(1); - - expect(configHelper.getEffectiveConfigProperties(EasyMock.anyObject(Cluster.class), EasyMock.anyObject(Map.class))).andReturn( - new HashMap<String, Map<String, String>>() { - { - put("hive-site", configTagVersion2Properties); - } - }).times(2); - - expect(configHelper.getMergedConfig(EasyMock.anyObject(Map.class), - EasyMock.anyObject(Map.class))).andReturn(new HashMap<String, String>()).anyTimes(); - - EasyMock.replay(configHelper); - // create an injector which will inject the mocks - injector = Guice.createInjector(Modules.override(new InMemoryDefaultTestModule()).with(new MockModule())); + injector = Guice.createInjector(new InMemoryDefaultTestModule()); injector.getInstance(GuiceJpaInitializer.class); @@ -272,10 +209,6 @@ public class UpgradeResourceProviderHDP22Test { injector = null; } - private void setupConfigHelper() { - - } - /** * Tests upgrades from HDP-2.2.x to HDP-2.2.y * @@ -374,8 +307,8 @@ public class UpgradeResourceProviderHDP22Test { String executionCommandJson = new String(ece.getCommand()); Map<String, Object> commandMap = gson.<Map<String, Object>> fromJson(executionCommandJson, Map.class); - // ensure that the latest tag is being used and that "*" is forcing a - // refresh - this is absolutely required for upgrades + // ensure that the latest tag is being used - this is absolutely required + // for upgrades Set<String> roleCommandsThatMustHaveRefresh = new HashSet<String>(); roleCommandsThatMustHaveRefresh.add("SERVICE_CHECK"); roleCommandsThatMustHaveRefresh.add("RESTART"); @@ -385,15 +318,10 @@ public class UpgradeResourceProviderHDP22Test { if (roleCommandsThatMustHaveRefresh.contains(roleCommand)) { assertTrue(commandMap.containsKey(KeyNames.REFRESH_CONFIG_TAGS_BEFORE_EXECUTION)); Object object = commandMap.get(KeyNames.REFRESH_CONFIG_TAGS_BEFORE_EXECUTION); - assertTrue(object instanceof List); - - @SuppressWarnings("unchecked") - List<String> tags = (List<String>) commandMap.get(KeyNames.REFRESH_CONFIG_TAGS_BEFORE_EXECUTION); - assertEquals(1, tags.size()); - assertEquals("*", tags.get(0)); + assertTrue(Boolean.valueOf(object.toString())); ExecutionCommandWrapperFactory ecwFactory = injector.getInstance(ExecutionCommandWrapperFactory.class); - ExecutionCommandWrapper executionCommandWrapper = ecwFactory.createFromJson(executionCommandJson); + ExecutionCommandWrapper executionCommandWrapper = ecwFactory.createFromJson(executionCommandJson); ExecutionCommand executionCommand = executionCommandWrapper.getExecutionCommand(); Map<String, Map<String, String>> configurationTags = executionCommand.getConfigurationTags(); assertEquals(configTagVersion2, configurationTags.get("hive-site").get("tag")); @@ -410,17 +338,4 @@ public class UpgradeResourceProviderHDP22Test { private UpgradeResourceProvider createProvider(AmbariManagementController amc) { return new UpgradeResourceProvider(amc); } - - /** - * - */ - private class MockModule implements Module { - /** - * - */ - @Override - public void configure(Binder binder) { - binder.bind(ConfigHelper.class).toInstance(configHelper); - } - } }
