AMBARI-6061. OutOfMemoryError during host checks on 2k nodes cluster (dlysnichenko)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/9faeaf5a Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/9faeaf5a Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/9faeaf5a Branch: refs/heads/trunk Commit: 9faeaf5a525bfee0932c1a654b28651c0b1705dd Parents: db83ea2 Author: Lisnichenko Dmitro <dlysniche...@hortonworks.com> Authored: Tue Aug 26 15:02:41 2014 +0300 Committer: Lisnichenko Dmitro <dlysniche...@hortonworks.com> Committed: Tue Aug 26 17:24:48 2014 +0300 ---------------------------------------------------------------------- .../server/actionmanager/ActionScheduler.java | 33 +++++ .../ambari/server/actionmanager/Stage.java | 32 ++++- .../server/actionmanager/StageFactory.java | 4 +- .../ambari/server/agent/ExecutionCommand.java | 2 +- .../controller/AmbariActionExecutionHelper.java | 9 +- .../AmbariCustomCommandExecutionHelper.java | 34 ++--- .../AmbariManagementControllerImpl.java | 128 ++++++++++--------- .../ambari/server/orm/entities/StageEntity.java | 28 ++++ .../ambari/server/stageplanner/RoleGraph.java | 3 +- .../server/upgrade/UpgradeCatalog170.java | 7 +- .../apache/ambari/server/utils/StageUtils.java | 57 +++++---- .../main/resources/Ambari-DDL-MySQL-CREATE.sql | 2 +- .../main/resources/Ambari-DDL-Oracle-CREATE.sql | 2 +- .../resources/Ambari-DDL-Postgres-CREATE.sql | 2 +- .../Ambari-DDL-Postgres-EMBEDDED-CREATE.sql | 2 +- .../ExecutionCommandWrapperTest.java | 2 +- .../actionmanager/TestActionDBAccessorImpl.java | 20 +-- .../server/actionmanager/TestActionManager.java | 31 +++-- .../actionmanager/TestActionScheduler.java | 33 +++-- .../ambari/server/actionmanager/TestStage.java | 4 +- .../server/agent/TestHeartbeatHandler.java | 15 ++- .../AmbariManagementControllerTest.java | 84 ++++++++---- .../server/stageplanner/TestStagePlanner.java | 6 +- .../server/upgrade/UpgradeCatalog170Test.java | 30 +++++ .../ambari/server/utils/TestStageUtils.java | 13 +- .../app/controllers/wizard/step3_controller.js | 1 - 26 files changed, 381 insertions(+), 203 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/9faeaf5a/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ActionScheduler.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ActionScheduler.java b/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ActionScheduler.java index b9a67b7..0385686 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ActionScheduler.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ActionScheduler.java @@ -121,6 +121,8 @@ class ActionScheduler implements Runnable { private boolean activeAwakeRequest = false; //Cache for clusterHostinfo, key - stageId-requestId private Cache<String, Map<String, Set<String>>> clusterHostInfoCache; + private Cache<String, Map<String, String>> commandParamsStageCache; + private Cache<String, Map<String, String>> hostParamsStageCache; public ActionScheduler(long sleepTimeMilliSec, long actionTimeoutMilliSec, ActionDBAccessor db, ActionQueue actionQueue, Clusters fsmObject, @@ -138,6 +140,12 @@ class ActionScheduler implements Runnable { this.clusterHostInfoCache = CacheBuilder.newBuilder(). expireAfterAccess(5, TimeUnit.MINUTES). build(); + this.commandParamsStageCache = CacheBuilder.newBuilder(). + expireAfterAccess(5, TimeUnit.MINUTES). + build(); + this.hostParamsStageCache = CacheBuilder.newBuilder(). + expireAfterAccess(5, TimeUnit.MINUTES). + build(); this.configuration = configuration; } @@ -748,6 +756,31 @@ class ActionScheduler implements Runnable { } cmd.setClusterHostInfo(clusterHostInfo); + + //Try to get commandParams from cache and merge them with command-level parameters + Map<String, String> commandParams = commandParamsStageCache.getIfPresent(stagePk); + + if (commandParams == null){ + Type type = new TypeToken<Map<String, String>>() {}.getType(); + commandParams = StageUtils.getGson().fromJson(s.getCommandParamsStage(), type); + commandParamsStageCache.put(stagePk, commandParams); + } + Map<String, String> commandParamsCmd = cmd.getCommandParams(); + commandParamsCmd.putAll(commandParams); + cmd.setCommandParams(commandParamsCmd); + + + //Try to get hostParams from cache and merge them with command-level parameters + Map<String, String> hostParams = hostParamsStageCache.getIfPresent(stagePk); + if (hostParams == null) { + Type type = new TypeToken<Map<String, String>>() {}.getType(); + hostParams = StageUtils.getGson().fromJson(s.getHostParamsStage(), type); + hostParamsStageCache.put(stagePk, hostParams); + } + Map<String, String> hostParamsCmd = cmd.getHostLevelParams(); + hostParamsCmd.putAll(hostParams); + cmd.setHostLevelParams(hostParamsCmd); + commandsToUpdate.add(cmd); } http://git-wip-us.apache.org/repos/asf/ambari/blob/9faeaf5a/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/Stage.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/Stage.java b/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/Stage.java index c4bbb46..bbc5ac3 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/Stage.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/Stage.java @@ -39,7 +39,6 @@ import org.apache.ambari.server.orm.entities.StageEntity; import org.apache.ambari.server.serveraction.ServerAction; import org.apache.ambari.server.state.Clusters; import org.apache.ambari.server.state.ServiceComponentHostEvent; -import org.apache.ambari.server.state.fsm.event.Event; import org.apache.ambari.server.state.svccomphost.ServiceComponentHostUpgradeEvent; import org.apache.ambari.server.utils.StageUtils; import org.apache.commons.lang.StringUtils; @@ -63,6 +62,8 @@ public class Stage { private final String logDir; private final String requestContext; private String clusterHostInfo; + private String commandParamsStage; + private String hostParamsStage; private int stageTimeout = -1; @@ -83,7 +84,9 @@ public class Stage { @Assisted("clusterName") @Nullable String clusterName, @Assisted("clusterId") long clusterId, @Assisted("requestContext") @Nullable String requestContext, - @Assisted("clusterHostInfo") String clusterHostInfo) { + @Assisted("clusterHostInfo") String clusterHostInfo, + @Assisted("commandParamsStage") String commandParamsStage, + @Assisted("hostParamsStage") String hostParamsStage) { this.wrappersLoaded = true; this.requestId = requestId; this.logDir = logDir; @@ -91,6 +94,8 @@ public class Stage { this.clusterId = clusterId; this.requestContext = requestContext == null ? "" : requestContext; this.clusterHostInfo = clusterHostInfo; + this.commandParamsStage = commandParamsStage; + this.hostParamsStage = hostParamsStage; } @AssistedInject @@ -113,6 +118,8 @@ public class Stage { requestContext = stageEntity.getRequestContext(); clusterHostInfo = stageEntity.getClusterHostInfo(); + commandParamsStage = stageEntity.getCommandParamsStage(); + hostParamsStage = stageEntity.getHostParamsStage(); List<Long> taskIds = hostRoleCommandDAO.findTaskIdsByStage(requestId, stageId); @@ -147,6 +154,8 @@ public class Stage { stageEntity.setHostRoleCommands(new ArrayList<HostRoleCommandEntity>()); stageEntity.setRoleSuccessCriterias(new ArrayList<RoleSuccessCriteriaEntity>()); stageEntity.setClusterHostInfo(clusterHostInfo); + stageEntity.setCommandParamsStage(commandParamsStage); + stageEntity.setHostParamsStage(hostParamsStage); for (Role role : successFactors.keySet()) { RoleSuccessCriteriaEntity roleSuccessCriteriaEntity = new RoleSuccessCriteriaEntity(); @@ -198,6 +207,23 @@ public class Stage { public void setClusterHostInfo(String clusterHostInfo) { this.clusterHostInfo = clusterHostInfo; } + + public String getCommandParamsStage() { + return commandParamsStage; + } + + public void setCommandParamsStage(String commandParamsStage) { + this.commandParamsStage = commandParamsStage; + } + + public String getHostParamsStage() { + return hostParamsStage; + } + + public void setHostParamsStage(String hostParamsStage) { + this.hostParamsStage = hostParamsStage; + } + public synchronized void setStageId(long stageId) { if (this.stageId != -1) { @@ -539,6 +565,8 @@ public class Stage { builder.append("logDir=" + logDir+"\n"); builder.append("requestContext="+requestContext+"\n"); builder.append("clusterHostInfo="+clusterHostInfo+"\n"); + builder.append("commandParamsStage="+commandParamsStage+"\n"); + builder.append("hostParamsStage="+hostParamsStage+"\n"); builder.append("Success Factors:\n"); for (Role r : successFactors.keySet()) { builder.append(" role: "+r+", factor: "+successFactors.get(r)+"\n"); http://git-wip-us.apache.org/repos/asf/ambari/blob/9faeaf5a/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/StageFactory.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/StageFactory.java b/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/StageFactory.java index fab2e96..a88558c 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/StageFactory.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/StageFactory.java @@ -29,7 +29,9 @@ public interface StageFactory { @Assisted("clusterName") String clusterName, @Assisted("clusterId") long clusterId, @Assisted("requestContext") String requestContext, - @Assisted("clusterHostInfo") String clusterHostInfo); + @Assisted("clusterHostInfo") String clusterHostInfo, + @Assisted("commandParamsStage") String commandParamsStage, + @Assisted("hostParamsStage") String hostParamsStage); Stage createExisting(StageEntity stageEntity); } http://git-wip-us.apache.org/repos/asf/ambari/blob/9faeaf5a/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java b/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java index fdf96df..33c5af4 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java @@ -57,7 +57,7 @@ public class ExecutionCommand extends AgentCommand { private Map<String, Map<String, Map<String, String>>> configurationAttributes; private Map<String, Map<String, String>> configurationTags; private Set<String> forceRefreshConfigTags = new HashSet<String>(); - private Map<String, String> commandParams; + private Map<String, String> commandParams = new HashMap<String, String>(); private String serviceName; private String componentName; http://git-wip-us.apache.org/repos/asf/ambari/blob/9faeaf5a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariActionExecutionHelper.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariActionExecutionHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariActionExecutionHelper.java index 64f132e..efe85e0 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariActionExecutionHelper.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariActionExecutionHelper.java @@ -20,7 +20,6 @@ package org.apache.ambari.server.controller; import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.COMMAND_TIMEOUT; import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.COMPONENT_CATEGORY; -import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.JDK_LOCATION; import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.SCRIPT; import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.SCRIPT_TYPE; @@ -206,12 +205,10 @@ public class AmbariActionExecutionHelper { * Add tasks to the stage based on the requested action execution * @param actionContext the context associated with the action * @param stage stage into which tasks must be inserted - * @param hostLevelParams host level params to send with the command * @throws AmbariException */ public void addExecutionCommandsToStage( - final ActionExecutionContext actionContext, - Stage stage, Map<String, String> hostLevelParams) + final ActionExecutionContext actionContext, Stage stage) throws AmbariException { String actionName = actionContext.getActionName(); @@ -337,9 +334,8 @@ public class AmbariActionExecutionHelper { configTags = managementController.findConfigurationTagsWithOverrides(cluster, hostName); } - Map<String, String> commandParams = actionContext.getParameters(); + Map<String, String> commandParams = new TreeMap<String, String>(); commandParams.put(COMMAND_TIMEOUT, actionContext.getTimeout().toString()); - commandParams.put(JDK_LOCATION, managementController.getJdkResourceUrl()); commandParams.put(SCRIPT, actionName + ".py"); commandParams.put(SCRIPT_TYPE, TYPE_PYTHON); @@ -353,7 +349,6 @@ public class AmbariActionExecutionHelper { execCmd.setConfigurations(configurations); execCmd.setConfigurationAttributes(configurationAttributes); execCmd.setConfigurationTags(configTags); - execCmd.setHostLevelParams(hostLevelParams); execCmd.setCommandParams(commandParams); execCmd.setServiceName(serviceName == null || serviceName.isEmpty() ? resourceFilter.getServiceName() : serviceName); http://git-wip-us.apache.org/repos/asf/ambari/blob/9faeaf5a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java index 339194f..156427d 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java @@ -22,7 +22,6 @@ import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.COMMAND_T import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.COMPONENT_CATEGORY; import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.CUSTOM_COMMAND; import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.HOOKS_FOLDER; -import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.JDK_LOCATION; import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.REPO_INFO; import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.SCRIPT; import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.SCRIPT_TYPE; @@ -204,7 +203,7 @@ public class AmbariCustomCommandExecutionHelper { private void addCustomCommandAction(final ActionExecutionContext actionExecutionContext, final RequestResourceFilter resourceFilter, - Stage stage, Map<String, String> hostLevelParams, + Stage stage, Map<String, String> additionalCommandParams, String commandDetail) throws AmbariException { @@ -290,6 +289,8 @@ public class AmbariCustomCommandExecutionHelper { if(actionExecutionContext.getParameters() != null && actionExecutionContext.getParameters().containsKey(KeyNames.REFRESH_ADITIONAL_COMPONENT_TAGS)){ execCmd.setForceRefreshConfigTags(parseAndValidateComponentsMapping(actionExecutionContext.getParameters().get(KeyNames.REFRESH_ADITIONAL_COMPONENT_TAGS))); } + + Map<String, String> hostLevelParams = new TreeMap<String, String>(); hostLevelParams.put(CUSTOM_COMMAND, commandName); // Set parameters required for re-installing clients on restart @@ -330,7 +331,6 @@ public class AmbariCustomCommandExecutionHelper { } commandParams.put(COMMAND_TIMEOUT, commandTimeout); - commandParams.put(JDK_LOCATION, managementController.getJdkResourceUrl()); commandParams.put(SERVICE_PACKAGE_FOLDER, serviceInfo.getServicePackageFolder()); @@ -357,7 +357,7 @@ public class AmbariCustomCommandExecutionHelper { /** * splits the passed commaseparated value and returns it as set - * @param comma separated list + * @param commaSeparatedTags separated list * @return set of items or null * @throws AmbariException */ @@ -372,7 +372,7 @@ public class AmbariCustomCommandExecutionHelper { private void findHostAndAddServiceCheckAction( final ActionExecutionContext actionExecutionContext, final RequestResourceFilter resourceFilter, - Stage stage, Map<String, String> hostLevelParams) + Stage stage) throws AmbariException { String clusterName = actionExecutionContext.getClusterName(); @@ -438,8 +438,7 @@ public class AmbariCustomCommandExecutionHelper { } addServiceCheckAction(stage, hostName, smokeTestRole, nowTimestamp, - serviceName, componentName, actionParameters, - hostLevelParams); + serviceName, componentName, actionParameters); } /** @@ -452,8 +451,7 @@ public class AmbariCustomCommandExecutionHelper { long nowTimestamp, String serviceName, String componentName, - Map<String, String> actionParameters, - Map<String, String> hostLevelParams) + Map<String, String> actionParameters) throws AmbariException { String clusterName = stage.getClusterName(); @@ -496,11 +494,6 @@ public class AmbariCustomCommandExecutionHelper { execCmd.setClusterHostInfo( StageUtils.getClusterHostInfo(clusters.getHostsForCluster(clusterName), cluster)); - if (hostLevelParams == null) { - hostLevelParams = new TreeMap<String, String>(); - } - execCmd.setHostLevelParams(hostLevelParams); - Map<String, String> commandParams = new TreeMap<String, String>(); String commandTimeout = configs.getDefaultAgentTaskTimeout(); @@ -525,7 +518,6 @@ public class AmbariCustomCommandExecutionHelper { } commandParams.put(COMMAND_TIMEOUT, commandTimeout); - commandParams.put(JDK_LOCATION, managementController.getJdkResourceUrl()); commandParams.put(SERVICE_PACKAGE_FOLDER, serviceInfo.getServicePackageFolder()); @@ -557,7 +549,7 @@ public class AmbariCustomCommandExecutionHelper { */ private void addDecommissionAction(final ActionExecutionContext actionExecutionContext, final RequestResourceFilter resourceFilter, - Stage stage, Map<String, String> hostLevelParams) + Stage stage) throws AmbariException { String clusterName = actionExecutionContext.getClusterName(); @@ -741,7 +733,7 @@ public class AmbariCustomCommandExecutionHelper { if (!serviceName.equals(Service.Type.HBASE.name()) || hostName.equals(primaryCandidate)) { commandParams.put(UPDATE_EXCLUDE_FILE_ONLY, "false"); addCustomCommandAction(commandContext, commandFilter, stage, - hostLevelParams, commandParams, commandDetail.toString()); + commandParams, commandDetail.toString()); } } } @@ -800,12 +792,10 @@ public class AmbariCustomCommandExecutionHelper { * Other than Service_Check and Decommission all other commands are pass-through * @param actionExecutionContext received request to execute a command * @param stage the initial stage for task creation - * @param hostLevelParams specific parameters for the hosts * @throws AmbariException */ public void addExecutionCommandsToStage(ActionExecutionContext actionExecutionContext, Stage stage, - Map<String, String> hostLevelParams, Map<String, String> requestParams) throws AmbariException { @@ -819,9 +809,9 @@ public class AmbariCustomCommandExecutionHelper { if (actionExecutionContext.getActionName().contains(SERVICE_CHECK_COMMAND_NAME)) { findHostAndAddServiceCheckAction(actionExecutionContext, - resourceFilter, stage, hostLevelParams); + resourceFilter, stage); } else if (actionExecutionContext.getActionName().equals(DECOMMISSION_COMMAND_NAME)) { - addDecommissionAction(actionExecutionContext, resourceFilter, stage, hostLevelParams); + addDecommissionAction(actionExecutionContext, resourceFilter, stage); } else if (isValidCustomCommand(actionExecutionContext, resourceFilter)) { String commandDetail = getReadableCustomCommandDetail(actionExecutionContext, resourceFilter); @@ -838,7 +828,7 @@ public class AmbariCustomCommandExecutionHelper { actionExecutionContext.getParameters().put(KeyNames.REFRESH_ADITIONAL_COMPONENT_TAGS, requestParams.get(KeyNames.REFRESH_ADITIONAL_COMPONENT_TAGS)); } addCustomCommandAction(actionExecutionContext, resourceFilter, stage, - hostLevelParams, extraParams, commandDetail); + extraParams, commandDetail); } else { throw new AmbariException("Unsupported action " + actionExecutionContext.getActionName()); http://git-wip-us.apache.org/repos/asf/ambari/blob/9faeaf5a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java index 3b80515..c465189 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java @@ -18,13 +18,48 @@ package org.apache.ambari.server.controller; -import com.google.common.cache.Cache; -import com.google.common.cache.CacheBuilder; -import com.google.gson.Gson; -import com.google.inject.Inject; -import com.google.inject.Injector; -import com.google.inject.Singleton; -import com.google.inject.persist.Transactional; +import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.AMBARI_DB_RCA_DRIVER; +import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.AMBARI_DB_RCA_PASSWORD; +import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.AMBARI_DB_RCA_URL; +import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.AMBARI_DB_RCA_USERNAME; +import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.COMMAND_TIMEOUT; +import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.DB_DRIVER_FILENAME; +import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.DB_NAME; +import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.HOOKS_FOLDER; +import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.JAVA_HOME; +import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.JCE_NAME; +import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.JDK_LOCATION; +import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.JDK_NAME; +import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.MYSQL_JDBC_URL; +import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.ORACLE_JDBC_URL; +import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.PACKAGE_LIST; +import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.REPO_INFO; +import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.SCRIPT; +import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.SCRIPT_TYPE; +import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.SERVICE_PACKAGE_FOLDER; +import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.SERVICE_REPO_INFO; +import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.STACK_NAME; +import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.STACK_VERSION; + +import java.io.File; +import java.io.IOException; +import java.net.InetAddress; +import java.text.MessageFormat; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.EnumMap; +import java.util.EnumSet; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; +import java.util.TreeMap; +import java.util.concurrent.TimeUnit; + import org.apache.ambari.server.AmbariException; import org.apache.ambari.server.ClusterNotFoundException; import org.apache.ambari.server.DuplicateResourceException; @@ -101,46 +136,14 @@ import org.apache.commons.lang.math.NumberUtils; import org.apache.http.client.utils.URIBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.File; -import java.io.IOException; -import java.net.InetAddress; -import java.text.MessageFormat; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.EnumMap; -import java.util.EnumSet; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.Set; -import java.util.TreeMap; -import java.util.concurrent.TimeUnit; -import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.AMBARI_DB_RCA_DRIVER; -import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.AMBARI_DB_RCA_PASSWORD; -import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.AMBARI_DB_RCA_URL; -import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.AMBARI_DB_RCA_USERNAME; -import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.COMMAND_TIMEOUT; -import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.DB_DRIVER_FILENAME; -import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.DB_NAME; -import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.HOOKS_FOLDER; -import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.JAVA_HOME; -import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.JCE_NAME; -import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.JDK_LOCATION; -import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.JDK_NAME; -import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.MYSQL_JDBC_URL; -import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.ORACLE_JDBC_URL; -import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.PACKAGE_LIST; -import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.REPO_INFO; -import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.SCRIPT; -import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.SCRIPT_TYPE; -import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.SERVICE_PACKAGE_FOLDER; -import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.SERVICE_REPO_INFO; -import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.STACK_NAME; -import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.STACK_VERSION; + +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; +import com.google.gson.Gson; +import com.google.inject.Inject; +import com.google.inject.Injector; +import com.google.inject.Singleton; +import com.google.inject.persist.Transactional; @Singleton public class AmbariManagementControllerImpl implements AmbariManagementController { @@ -798,13 +801,16 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle } } - private Stage createNewStage(long id, Cluster cluster, long requestId, String requestContext, String clusterHostInfo) { + private Stage createNewStage(long id, Cluster cluster, long requestId, + String requestContext, String clusterHostInfo, + String commandParamsStage, String hostParamsStage) { String logDir = BASE_LOG_DIR + File.pathSeparator + requestId; Stage stage = stageFactory.createNew(requestId, logDir, null == cluster ? null : cluster.getClusterName(), null == cluster ? -1L : cluster.getClusterId(), - requestContext, clusterHostInfo); + requestContext, clusterHostInfo, commandParamsStage, + hostParamsStage); stage.setStageId(id); return stage; } @@ -1582,7 +1588,7 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle + ", repoInfo=" + repoInfo); } - Map<String, String> hostParams = createDefaultHostParams(cluster); + Map<String, String> hostParams = new TreeMap<String, String>(); hostParams.put(REPO_INFO, repoInfo); hostParams.putAll(getRcaParameters()); @@ -1731,9 +1737,11 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle clusters.getHostsForCluster(cluster.getClusterName()), cluster); String clusterHostInfoJson = StageUtils.getGson().toJson(clusterHostInfo); + String HostParamsJson = StageUtils.getGson().toJson(createDefaultHostParams(cluster)); Stage stage = createNewStage(requestStages.getLastStageId() + 1, cluster, - requestStages.getId(), requestProperties.get(REQUEST_CONTEXT_PROPERTY), clusterHostInfoJson); + requestStages.getId(), requestProperties.get(REQUEST_CONTEXT_PROPERTY), + clusterHostInfoJson, "{}", HostParamsJson); //HACK String jobtrackerHost = getJobTrackerHost(cluster); @@ -1945,7 +1953,7 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle customCommandExecutionHelper.addServiceCheckAction(stage, clientHost, smokeTestRole, nowTimestamp, serviceName, - null, null, createDefaultHostParams(cluster)); + null, null); } RoleCommandOrder rco = getRoleCommandOrder(cluster); @@ -2947,24 +2955,28 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle actionExecutionHelper.validateAction(actionRequest); } - Map<String, String> params = new HashMap<String, String>(); + Map<String, String> commandParamsStage = StageUtils.getCommandParamsStage(actionExecContext); + Map<String, String> hostParamsStage = new HashMap<String, String>(); Map<String, Set<String>> clusterHostInfo; String clusterHostInfoJson = "{}"; if (null != cluster) { clusterHostInfo = StageUtils.getClusterHostInfo( clusters.getHostsForCluster(cluster.getClusterName()), cluster); - params = createDefaultHostParams(cluster); + hostParamsStage = createDefaultHostParams(cluster); clusterHostInfoJson = StageUtils.getGson().toJson(clusterHostInfo); } - Stage stage = createNewStage(0, cluster, actionManager.getNextRequestId(), requestContext, clusterHostInfoJson); + String hostParamsStageJson = StageUtils.getGson().toJson(hostParamsStage); + String commandParamsStageJson = StageUtils.getGson().toJson(commandParamsStage); + + Stage stage = createNewStage(0, cluster, actionManager.getNextRequestId(), requestContext, + clusterHostInfoJson, commandParamsStageJson, hostParamsStageJson); if (actionRequest.isCommand()) { - customCommandExecutionHelper.addExecutionCommandsToStage(actionExecContext, stage, - params, requestProperties); + customCommandExecutionHelper.addExecutionCommandsToStage(actionExecContext, stage, requestProperties); } else { - actionExecutionHelper.addExecutionCommandsToStage(actionExecContext, stage, params); + actionExecutionHelper.addExecutionCommandsToStage(actionExecContext, stage); } RoleGraph rg; http://git-wip-us.apache.org/repos/asf/ambari/blob/9faeaf5a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/StageEntity.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/StageEntity.java b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/StageEntity.java index 245c9e9..a7bc948 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/StageEntity.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/StageEntity.java @@ -51,6 +51,14 @@ public class StageEntity { @Column(name = "cluster_host_info") @Basic private byte[] clusterHostInfo; + + @Column(name = "command_params") + @Basic + private byte[] commandParamsStage; + + @Column(name = "host_params") + @Basic + private byte[] hostParamsStage; @ManyToOne @JoinColumn(name = "request_id", referencedColumnName = "request_id", nullable = false) @@ -106,6 +114,22 @@ public class StageEntity { public void setClusterHostInfo(String clusterHostInfo) { this.clusterHostInfo = clusterHostInfo.getBytes(); } + + public String getCommandParamsStage() { + return commandParamsStage == null ? new String() : new String(commandParamsStage); + } + + public void setCommandParamsStage(String commandParamsStage) { + this.commandParamsStage = commandParamsStage.getBytes(); + } + + public String getHostParamsStage() { + return hostParamsStage == null ? new String() : new String(hostParamsStage); + } + + public void setHostParamsStage(String hostParamsStage) { + this.hostParamsStage = hostParamsStage.getBytes(); + } public void setRequestContext(String requestContext) { if (requestContext != null) { @@ -125,6 +149,8 @@ public class StageEntity { if (requestId != null ? !requestId.equals(that.requestId) : that.requestId != null) return false; if (stageId != null ? !stageId.equals(that.stageId) : that.stageId != null) return false; if (clusterHostInfo != null ? !clusterHostInfo.equals(that.clusterHostInfo) : that.clusterHostInfo != null) return false; + if (commandParamsStage != null ? !commandParamsStage.equals(that.commandParamsStage) : that.commandParamsStage != null) return false; + if (hostParamsStage != null ? !hostParamsStage.equals(that.hostParamsStage) : that.hostParamsStage != null) return false; return !(requestContext != null ? !requestContext.equals(that.requestContext) : that.requestContext != null); } @@ -136,6 +162,8 @@ public class StageEntity { result = 31 * result + (stageId != null ? stageId.hashCode() : 0); result = 31 * result + (logInfo != null ? logInfo.hashCode() : 0); result = 31 * result + (clusterHostInfo != null ? clusterHostInfo.hashCode() : 0); + result = 31 * result + (commandParamsStage != null ? commandParamsStage.hashCode() : 0); + result = 31 * result + (hostParamsStage != null ? hostParamsStage.hashCode() : 0); result = 31 * result + (requestContext != null ? requestContext.hashCode() : 0); return result; } http://git-wip-us.apache.org/repos/asf/ambari/blob/9faeaf5a/ambari-server/src/main/java/org/apache/ambari/server/stageplanner/RoleGraph.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/stageplanner/RoleGraph.java b/ambari-server/src/main/java/org/apache/ambari/server/stageplanner/RoleGraph.java index 913b878..4fe3787 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/stageplanner/RoleGraph.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/stageplanner/RoleGraph.java @@ -139,7 +139,8 @@ public class RoleGraph { Stage newStage = new Stage(origStage.getRequestId(), origStage.getLogDir(), origStage.getClusterName(), origStage.getClusterId(), - origStage.getRequestContext(), origStage.getClusterHostInfo()); + origStage.getRequestContext(), origStage.getClusterHostInfo(), + origStage.getCommandParamsStage(), origStage.getHostParamsStage()); newStage.setSuccessFactors(origStage.getSuccessFactors()); for (RoleGraphNode rgn : stageGraphNodes) { for (String host : rgn.getHosts()) { http://git-wip-us.apache.org/repos/asf/ambari/blob/9faeaf5a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog170.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog170.java b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog170.java index 30059ac..dc3d5b8 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog170.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog170.java @@ -73,7 +73,6 @@ import org.apache.ambari.server.state.Cluster; import org.apache.ambari.server.state.Clusters; import org.apache.ambari.server.state.Config; import org.apache.ambari.server.state.ConfigHelper; -import org.apache.ambari.server.view.configuration.InstanceConfig; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -249,6 +248,12 @@ public class UpgradeCatalog170 extends AbstractUpgradeCatalog { dbAccessor.addColumn("host_role_command", new DBColumnInfo("output_log", String.class, 255, null, true)); + + dbAccessor.addColumn("stage", new DBColumnInfo("command_params", + byte[].class, null, null, true)); + dbAccessor.addColumn("stage", new DBColumnInfo("host_params", + byte[].class, null, null, true)); + dbAccessor.addColumn("host_role_command", new DBColumnInfo("error_log", String.class, 255, null, true)); http://git-wip-us.apache.org/repos/asf/ambari/blob/9faeaf5a/ambari-server/src/main/java/org/apache/ambari/server/utils/StageUtils.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/utils/StageUtils.java b/ambari-server/src/main/java/org/apache/ambari/server/utils/StageUtils.java index 7160bcf..2a13653 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/utils/StageUtils.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/utils/StageUtils.java @@ -17,14 +17,32 @@ */ package org.apache.ambari.server.utils; -import com.google.common.base.Joiner; -import com.google.gson.Gson; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.nio.charset.Charset; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; +import java.util.SortedSet; +import java.util.TreeMap; +import java.util.TreeSet; + +import javax.xml.bind.JAXBException; + import org.apache.ambari.server.AmbariException; import org.apache.ambari.server.Role; import org.apache.ambari.server.RoleCommand; import org.apache.ambari.server.actionmanager.Stage; import org.apache.ambari.server.agent.ExecutionCommand; -import org.apache.ambari.server.configuration.Configuration; +import org.apache.ambari.server.controller.ActionExecutionContext; import org.apache.ambari.server.state.Cluster; import org.apache.ambari.server.state.Host; import org.apache.ambari.server.state.HostComponentAdminState; @@ -40,25 +58,8 @@ import org.codehaus.jackson.map.JsonMappingException; import org.codehaus.jackson.map.ObjectMapper; import org.codehaus.jackson.map.SerializationConfig; -import javax.xml.bind.JAXBException; -import java.io.ByteArrayInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.net.InetAddress; -import java.net.UnknownHostException; -import java.nio.charset.Charset; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.Set; -import java.util.SortedSet; -import java.util.TreeMap; -import java.util.TreeSet; +import com.google.common.base.Joiner; +import com.google.gson.Gson; public class StageUtils { @@ -148,20 +149,20 @@ public class StageUtils { return requestStageIds; } - public static Stage getATestStage(long requestId, long stageId, String clusterHostInfo) { + public static Stage getATestStage(long requestId, long stageId, String clusterHostInfo, String commandParamsStage, String hostParamsStage) { String hostname; try { hostname = InetAddress.getLocalHost().getHostName(); } catch (UnknownHostException e) { hostname = "host-dummy"; } - return getATestStage(requestId, stageId, hostname, clusterHostInfo); + return getATestStage(requestId, stageId, hostname, clusterHostInfo, commandParamsStage, hostParamsStage); } //For testing only - public static Stage getATestStage(long requestId, long stageId, String hostname, String clusterHostInfo) { + public static Stage getATestStage(long requestId, long stageId, String hostname, String clusterHostInfo, String commandParamsStage, String hostParamsStage) { - Stage s = new Stage(requestId, "/tmp", "cluster1", 1L, "context", clusterHostInfo); + Stage s = new Stage(requestId, "/tmp", "cluster1", 1L, "context", clusterHostInfo, commandParamsStage, hostParamsStage); s.setStageId(stageId); long now = System.currentTimeMillis(); s.addHostRoleExecutionCommand(hostname, Role.NAMENODE, RoleCommand.INSTALL, @@ -221,6 +222,10 @@ public class StageUtils { InputStream is = new ByteArrayInputStream(json.getBytes(Charset.forName("UTF8"))); return mapper.readValue(is, clazz); } + + public static Map<String, String> getCommandParamsStage(ActionExecutionContext actionExecContext) throws AmbariException { + return actionExecContext.getParameters() != null ? actionExecContext.getParameters() : new TreeMap<String, String>(); + } public static Map<String, Set<String>> getClusterHostInfo( Map<String, Host> allHosts, Cluster cluster) throws AmbariException { http://git-wip-us.apache.org/repos/asf/ambari/blob/9faeaf5a/ambari-server/src/main/resources/Ambari-DDL-MySQL-CREATE.sql ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/Ambari-DDL-MySQL-CREATE.sql b/ambari-server/src/main/resources/Ambari-DDL-MySQL-CREATE.sql index 29fa041..2d83aeb 100644 --- a/ambari-server/src/main/resources/Ambari-DDL-MySQL-CREATE.sql +++ b/ambari-server/src/main/resources/Ambari-DDL-MySQL-CREATE.sql @@ -45,7 +45,7 @@ CREATE TABLE members (member_id INTEGER, group_id INTEGER NOT NULL, user_id INTE CREATE TABLE execution_command (task_id BIGINT NOT NULL, command LONGBLOB, PRIMARY KEY (task_id)); CREATE TABLE host_role_command (task_id BIGINT NOT NULL, attempt_count SMALLINT NOT NULL, event LONGTEXT NOT NULL, exitcode INTEGER NOT NULL, host_name VARCHAR(255) NOT NULL, last_attempt_time BIGINT NOT NULL, request_id BIGINT NOT NULL, role VARCHAR(255), role_command VARCHAR(255), stage_id BIGINT NOT NULL, start_time BIGINT NOT NULL, end_time BIGINT, status VARCHAR(255), std_error LONGBLOB, std_out LONGBLOB, output_log VARCHAR(255) NULL, error_log VARCHAR(255) NULL, structured_out LONGBLOB, command_detail VARCHAR(255), custom_command_name VARCHAR(255), PRIMARY KEY (task_id)); CREATE TABLE role_success_criteria (role VARCHAR(255) NOT NULL, request_id BIGINT NOT NULL, stage_id BIGINT NOT NULL, success_factor DOUBLE NOT NULL, PRIMARY KEY (role, request_id, stage_id)); -CREATE TABLE stage (stage_id BIGINT NOT NULL, request_id BIGINT NOT NULL, cluster_id BIGINT, log_info VARCHAR(255) NOT NULL, request_context VARCHAR(255), cluster_host_info LONGBLOB, PRIMARY KEY (stage_id, request_id)); +CREATE TABLE stage (stage_id BIGINT NOT NULL, request_id BIGINT NOT NULL, cluster_id BIGINT, log_info VARCHAR(255) NOT NULL, request_context VARCHAR(255), cluster_host_info LONGBLOB, command_params LONGBLOB, host_params LONGBLOB, PRIMARY KEY (stage_id, request_id)); CREATE TABLE request (request_id BIGINT NOT NULL, cluster_id BIGINT, request_schedule_id BIGINT, command_name VARCHAR(255), create_time BIGINT NOT NULL, end_time BIGINT NOT NULL, inputs LONGBLOB, request_context VARCHAR(255), request_type VARCHAR(255), start_time BIGINT NOT NULL, status VARCHAR(255), PRIMARY KEY (request_id)); CREATE TABLE requestresourcefilter (filter_id BIGINT NOT NULL, request_id BIGINT NOT NULL, service_name VARCHAR(255), component_name VARCHAR(255), hosts LONGBLOB, PRIMARY KEY (filter_id)); CREATE TABLE requestoperationlevel (operation_level_id BIGINT NOT NULL, request_id BIGINT NOT NULL, level_name VARCHAR(255), cluster_name VARCHAR(255), service_name VARCHAR(255), host_component_name VARCHAR(255), host_name VARCHAR(255), PRIMARY KEY (operation_level_id)); http://git-wip-us.apache.org/repos/asf/ambari/blob/9faeaf5a/ambari-server/src/main/resources/Ambari-DDL-Oracle-CREATE.sql ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/Ambari-DDL-Oracle-CREATE.sql b/ambari-server/src/main/resources/Ambari-DDL-Oracle-CREATE.sql index 457404d..241bb2b 100644 --- a/ambari-server/src/main/resources/Ambari-DDL-Oracle-CREATE.sql +++ b/ambari-server/src/main/resources/Ambari-DDL-Oracle-CREATE.sql @@ -36,7 +36,7 @@ CREATE TABLE members (member_id NUMBER(10), group_id NUMBER(10) NOT NULL, user_i CREATE TABLE execution_command (task_id NUMBER(19) NOT NULL, command BLOB NULL, PRIMARY KEY (task_id)); CREATE TABLE host_role_command (task_id NUMBER(19) NOT NULL, attempt_count NUMBER(5) NOT NULL, event CLOB NULL, exitcode NUMBER(10) NOT NULL, host_name VARCHAR2(255) NOT NULL, last_attempt_time NUMBER(19) NOT NULL, request_id NUMBER(19) NOT NULL, role VARCHAR2(255) NULL, role_command VARCHAR2(255) NULL, stage_id NUMBER(19) NOT NULL, start_time NUMBER(19) NOT NULL, end_time NUMBER(19), status VARCHAR2(255) NULL, std_error BLOB NULL, std_out BLOB NULL, output_log VARCHAR2(255) NULL, error_log VARCHAR2(255) NULL, structured_out BLOB NULL, command_detail VARCHAR2(255) NULL, custom_command_name VARCHAR2(255) NULL, PRIMARY KEY (task_id)); CREATE TABLE role_success_criteria (role VARCHAR2(255) NOT NULL, request_id NUMBER(19) NOT NULL, stage_id NUMBER(19) NOT NULL, success_factor NUMBER(19,4) NOT NULL, PRIMARY KEY (role, request_id, stage_id)); -CREATE TABLE stage (stage_id NUMBER(19) NOT NULL, request_id NUMBER(19) NOT NULL, cluster_id NUMBER(19) NULL, log_info VARCHAR2(255) NULL, request_context VARCHAR2(255) NULL, cluster_host_info BLOB NOT NULL, PRIMARY KEY (stage_id, request_id)); +CREATE TABLE stage (stage_id NUMBER(19) NOT NULL, request_id NUMBER(19) NOT NULL, cluster_id NUMBER(19) NULL, log_info VARCHAR2(255) NULL, request_context VARCHAR2(255) NULL, cluster_host_info BLOB NOT NULL, command_params BLOB, host_params BLOB, PRIMARY KEY (stage_id, request_id)); CREATE TABLE request (request_id NUMBER(19) NOT NULL, cluster_id NUMBER(19), request_schedule_id NUMBER(19), command_name VARCHAR(255), create_time NUMBER(19) NOT NULL, end_time NUMBER(19) NOT NULL, inputs BLOB, request_context VARCHAR(255), request_type VARCHAR(255), start_time NUMBER(19) NOT NULL, status VARCHAR(255), PRIMARY KEY (request_id)); CREATE TABLE requestresourcefilter (filter_id NUMBER(19) NOT NULL, request_id NUMBER(19) NOT NULL, service_name VARCHAR2(255), component_name VARCHAR2(255), hosts BLOB, PRIMARY KEY (filter_id)); CREATE TABLE requestoperationlevel (operation_level_id NUMBER(19) NOT NULL, request_id NUMBER(19) NOT NULL, level_name VARCHAR2(255), cluster_name VARCHAR2(255), service_name VARCHAR2(255), host_component_name VARCHAR2(255), host_name VARCHAR2(255), PRIMARY KEY (operation_level_id)); http://git-wip-us.apache.org/repos/asf/ambari/blob/9faeaf5a/ambari-server/src/main/resources/Ambari-DDL-Postgres-CREATE.sql ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/Ambari-DDL-Postgres-CREATE.sql b/ambari-server/src/main/resources/Ambari-DDL-Postgres-CREATE.sql index b58fd21..ddea71c 100644 --- a/ambari-server/src/main/resources/Ambari-DDL-Postgres-CREATE.sql +++ b/ambari-server/src/main/resources/Ambari-DDL-Postgres-CREATE.sql @@ -57,7 +57,7 @@ CREATE TABLE host_role_command (task_id BIGINT NOT NULL, attempt_count SMALLINT CREATE TABLE role_success_criteria (role VARCHAR(255) NOT NULL, request_id BIGINT NOT NULL, stage_id BIGINT NOT NULL, success_factor FLOAT NOT NULL, PRIMARY KEY (role, request_id, stage_id)); -CREATE TABLE stage (stage_id BIGINT NOT NULL, request_id BIGINT NOT NULL, cluster_id BIGINT NOT NULL, log_info VARCHAR(255) NOT NULL, request_context VARCHAR(255), cluster_host_info BYTEA NOT NULL, PRIMARY KEY (stage_id, request_id)); +CREATE TABLE stage (stage_id BIGINT NOT NULL, request_id BIGINT NOT NULL, cluster_id BIGINT NOT NULL, log_info VARCHAR(255) NOT NULL, request_context VARCHAR(255), cluster_host_info BYTEA NOT NULL, command_params BYTEA, host_params BYTEA, PRIMARY KEY (stage_id, request_id)); CREATE TABLE request (request_id BIGINT NOT NULL, cluster_id BIGINT, command_name VARCHAR(255), create_time BIGINT NOT NULL, end_time BIGINT NOT NULL, inputs BYTEA, request_context VARCHAR(255), request_type VARCHAR(255), request_schedule_id BIGINT, start_time BIGINT NOT NULL, status VARCHAR(255), PRIMARY KEY (request_id)); http://git-wip-us.apache.org/repos/asf/ambari/blob/9faeaf5a/ambari-server/src/main/resources/Ambari-DDL-Postgres-EMBEDDED-CREATE.sql ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/Ambari-DDL-Postgres-EMBEDDED-CREATE.sql b/ambari-server/src/main/resources/Ambari-DDL-Postgres-EMBEDDED-CREATE.sql index 664bbe9..17b95c2 100644 --- a/ambari-server/src/main/resources/Ambari-DDL-Postgres-EMBEDDED-CREATE.sql +++ b/ambari-server/src/main/resources/Ambari-DDL-Postgres-EMBEDDED-CREATE.sql @@ -88,7 +88,7 @@ GRANT ALL PRIVILEGES ON TABLE ambari.host_role_command TO :username; CREATE TABLE ambari.role_success_criteria (role VARCHAR(255) NOT NULL, request_id BIGINT NOT NULL, stage_id BIGINT NOT NULL, success_factor FLOAT NOT NULL, PRIMARY KEY (role, request_id, stage_id)); GRANT ALL PRIVILEGES ON TABLE ambari.role_success_criteria TO :username; -CREATE TABLE ambari.stage (stage_id BIGINT NOT NULL, request_id BIGINT NOT NULL, cluster_id BIGINT NOT NULL, log_info VARCHAR(255) NOT NULL, request_context VARCHAR(255), cluster_host_info BYTEA NOT NULL, PRIMARY KEY (stage_id, request_id)); +CREATE TABLE ambari.stage (stage_id BIGINT NOT NULL, request_id BIGINT NOT NULL, cluster_id BIGINT NOT NULL, log_info VARCHAR(255) NOT NULL, request_context VARCHAR(255), cluster_host_info BYTEA NOT NULL, command_params BYTEA, host_params BYTEA, PRIMARY KEY (stage_id, request_id)); GRANT ALL PRIVILEGES ON TABLE ambari.stage TO :username; CREATE TABLE ambari.request (request_id BIGINT NOT NULL, cluster_id BIGINT, command_name VARCHAR(255), create_time BIGINT NOT NULL, end_time BIGINT NOT NULL, inputs BYTEA, request_context VARCHAR(255), request_type VARCHAR(255), request_schedule_id BIGINT, start_time BIGINT NOT NULL, status VARCHAR(255), PRIMARY KEY (request_id)); http://git-wip-us.apache.org/repos/asf/ambari/blob/9faeaf5a/ambari-server/src/test/java/org/apache/ambari/server/actionmanager/ExecutionCommandWrapperTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/actionmanager/ExecutionCommandWrapperTest.java b/ambari-server/src/test/java/org/apache/ambari/server/actionmanager/ExecutionCommandWrapperTest.java index 034ac5b..948f137 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/actionmanager/ExecutionCommandWrapperTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/actionmanager/ExecutionCommandWrapperTest.java @@ -155,7 +155,7 @@ public class ExecutionCommandWrapperTest { private static void createTask(ActionDBAccessor db, long requestId, long stageId, String hostName, String clusterName) throws AmbariException { - Stage s = new Stage(requestId, "/var/log", clusterName, 1L, "execution command wrapper test", "clusterHostInfo"); + Stage s = new Stage(requestId, "/var/log", clusterName, 1L, "execution command wrapper test", "clusterHostInfo", "commandParamsStage", "hostParamsStage"); s.setStageId(stageId); s.addHostRoleExecutionCommand(hostName, Role.NAMENODE, RoleCommand.START, http://git-wip-us.apache.org/repos/asf/ambari/blob/9faeaf5a/ambari-server/src/test/java/org/apache/ambari/server/actionmanager/TestActionDBAccessorImpl.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/actionmanager/TestActionDBAccessorImpl.java b/ambari-server/src/test/java/org/apache/ambari/server/actionmanager/TestActionDBAccessorImpl.java index 2850897..6d4f056 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/actionmanager/TestActionDBAccessorImpl.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/actionmanager/TestActionDBAccessorImpl.java @@ -16,15 +16,15 @@ * limitations under the License. */ package org.apache.ambari.server.actionmanager; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; import java.util.ArrayList; import java.util.Collections; import java.util.List; -import com.google.inject.persist.UnitOfWork; - -import junit.framework.Assert; - import org.apache.ambari.server.AmbariException; import org.apache.ambari.server.Role; import org.apache.ambari.server.RoleCommand; @@ -52,8 +52,9 @@ import com.google.inject.Guice; import com.google.inject.Inject; import com.google.inject.Injector; import com.google.inject.persist.PersistService; +import com.google.inject.persist.UnitOfWork; -import static org.junit.Assert.*; +import junit.framework.Assert; public class TestActionDBAccessorImpl { private static final Logger log = LoggerFactory.getLogger(TestActionDBAccessorImpl.class); @@ -348,7 +349,8 @@ public class TestActionDBAccessorImpl { @Test public void testAbortRequest() throws AmbariException { - Stage s = new Stage(requestId, "/a/b", "cluster1", 1L, "action db accessor test", "clusterHostInfo"); + Stage s = new Stage(requestId, "/a/b", "cluster1", 1L, "action db accessor test", + "clusterHostInfo", "commandParamsStage", "hostParamsStage"); s.setStageId(stageId); clusters.addHost("host2"); @@ -412,7 +414,8 @@ public class TestActionDBAccessorImpl { } private Stage createStubStage(String hostname, long requestId, long stageId) { - Stage s = new Stage(requestId, "/a/b", "cluster1", 1L, "action db accessor test", "clusterHostInfo"); + Stage s = new Stage(requestId, "/a/b", "cluster1", 1L, "action db accessor test", + "clusterHostInfo", "commandParamsStage", "hostParamsStage"); s.setStageId(stageId); s.addHostRoleExecutionCommand(hostname, Role.HBASE_MASTER, RoleCommand.START, @@ -429,7 +432,8 @@ public class TestActionDBAccessorImpl { private void populateActionDBWithCustomAction(ActionDBAccessor db, String hostname, long requestId, long stageId) throws AmbariException { - Stage s = new Stage(requestId, "/a/b", "cluster1", 1L, "action db accessor test", ""); + Stage s = new Stage(requestId, "/a/b", "cluster1", 1L, "action db accessor test", + "", "commandParamsStage", "hostParamsStage"); s.setStageId(stageId); s.addHostRoleExecutionCommand(hostname, Role.valueOf(actionName), RoleCommand.ACTIONEXECUTE, http://git-wip-us.apache.org/repos/asf/ambari/blob/9faeaf5a/ambari-server/src/test/java/org/apache/ambari/server/actionmanager/TestActionManager.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/actionmanager/TestActionManager.java b/ambari-server/src/test/java/org/apache/ambari/server/actionmanager/TestActionManager.java index 39bc762..5a2c467 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/actionmanager/TestActionManager.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/actionmanager/TestActionManager.java @@ -17,17 +17,24 @@ */ package org.apache.ambari.server.actionmanager; -import com.google.inject.Guice; -import com.google.inject.Injector; -import com.google.inject.persist.PersistService; -import com.google.inject.persist.UnitOfWork; -import junit.framework.Assert; +import static org.easymock.EasyMock.createNiceMock; +import static org.easymock.EasyMock.createStrictMock; +import static org.easymock.EasyMock.expect; +import static org.easymock.EasyMock.replay; +import static org.easymock.EasyMock.verify; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertSame; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + import org.apache.ambari.server.AmbariException; import org.apache.ambari.server.Role; import org.apache.ambari.server.RoleCommand; import org.apache.ambari.server.agent.ActionQueue; import org.apache.ambari.server.agent.CommandReport; -import org.apache.ambari.server.api.services.BaseRequest; import org.apache.ambari.server.controller.HostsMap; import org.apache.ambari.server.orm.GuiceJpaInitializer; import org.apache.ambari.server.orm.InMemoryDefaultTestModule; @@ -39,12 +46,12 @@ import org.junit.Before; import org.junit.Ignore; import org.junit.Test; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; +import com.google.inject.Guice; +import com.google.inject.Injector; +import com.google.inject.persist.PersistService; +import com.google.inject.persist.UnitOfWork; -import static org.easymock.EasyMock.*; -import static org.junit.Assert.*; +import junit.framework.Assert; public class TestActionManager { @@ -160,7 +167,7 @@ public class TestActionManager { } private void populateActionDB(ActionDBAccessor db, String hostname) throws AmbariException { - Stage s = new Stage(requestId, "/a/b", "cluster1", 1L, "action manager test", "clusterHostInfo"); + Stage s = new Stage(requestId, "/a/b", "cluster1", 1L, "action manager test", "clusterHostInfo", "commandParamsStage", "hostParamsStage"); s.setStageId(stageId); s.addHostRoleExecutionCommand(hostname, Role.HBASE_MASTER, RoleCommand.START, http://git-wip-us.apache.org/repos/asf/ambari/blob/9faeaf5a/ambari-server/src/test/java/org/apache/ambari/server/actionmanager/TestActionScheduler.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/actionmanager/TestActionScheduler.java b/ambari-server/src/test/java/org/apache/ambari/server/actionmanager/TestActionScheduler.java index a536bef..c4a88cf 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/actionmanager/TestActionScheduler.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/actionmanager/TestActionScheduler.java @@ -127,7 +127,8 @@ public class TestActionScheduler { ActionDBAccessor db = mock(ActionDBAccessorImpl.class); List<Stage> stages = new ArrayList<Stage>(); - Stage s = StageUtils.getATestStage(1, 977, hostname, CLUSTER_HOST_INFO); + Stage s = StageUtils.getATestStage(1, 977, hostname, CLUSTER_HOST_INFO, + "{\"host_param\":\"param_value\"}", "{\"stage_param\":\"param_value\"}"); stages.add(s); when(db.getStagesInProgress()).thenReturn(stages); @@ -207,7 +208,8 @@ public class TestActionScheduler { when(host.getHostName()).thenReturn(hostname); List<Stage> stages = new ArrayList<Stage>(); - final Stage s = StageUtils.getATestStage(1, 977, hostname, CLUSTER_HOST_INFO); + final Stage s = StageUtils.getATestStage(1, 977, hostname, CLUSTER_HOST_INFO, + "{\"host_param\":\"param_value\"}", "{\"stage_param\":\"param_value\"}"); stages.add(s); ActionDBAccessor db = mock(ActionDBAccessor.class); @@ -271,7 +273,8 @@ public class TestActionScheduler { when(host.getHostName()).thenReturn(hostname); List<Stage> stages = new ArrayList<Stage>(); - final Stage s = StageUtils.getATestStage(1, 977, hostname, CLUSTER_HOST_INFO); + final Stage s = StageUtils.getATestStage(1, 977, hostname, CLUSTER_HOST_INFO, + "{\"host_param\":\"param_value\"}", "{\"stage_param\":\"param_value\"}"); stages.add(s); ActionDBAccessor db = mock(ActionDBAccessor.class); @@ -349,7 +352,7 @@ public class TestActionScheduler { final List<Stage> stages = new ArrayList<Stage>(); Stage stage = new Stage(1, "/tmp", "cluster1", 1L, "stageWith2Tasks", - CLUSTER_HOST_INFO); + CLUSTER_HOST_INFO, "", ""); addInstallTaskToStage(stage, hostname1, "cluster1", Role.DATANODE, RoleCommand.INSTALL, Service.Type.HDFS, 1); addInstallTaskToStage(stage, hostname2, "cluster1", Role.NAMENODE, @@ -575,7 +578,8 @@ public class TestActionScheduler { private static Stage getStageWithServerAction(long requestId, long stageId, String hostName, Map<String, String> payload, String requestContext) { - Stage stage = new Stage(requestId, "/tmp", "cluster1", 1L, requestContext, CLUSTER_HOST_INFO); + Stage stage = new Stage(requestId, "/tmp", "cluster1", 1L, requestContext, CLUSTER_HOST_INFO, + "", ""); stage.setStageId(stageId); long now = System.currentTimeMillis(); stage.addServerActionCommand(ServerAction.Command.FINALIZE_UPGRADE, Role.AMBARI_SERVER_ACTION, @@ -989,7 +993,7 @@ public class TestActionScheduler { long now = System.currentTimeMillis(); Stage stage = new Stage(1, "/tmp", "cluster1", 1L, - "testRequestFailureBasedOnSuccessFactor", CLUSTER_HOST_INFO); + "testRequestFailureBasedOnSuccessFactor", CLUSTER_HOST_INFO, "", ""); stage.setStageId(1); addHostRoleExecutionCommand(now, stage, Role.SQOOP, Service.Type.SQOOP, @@ -1174,7 +1178,8 @@ public class TestActionScheduler { final List<Stage> stages = new ArrayList<Stage>(); long now = System.currentTimeMillis(); - Stage stage = new Stage(1, "/tmp", "cluster1", 1L, "testRequestFailureBasedOnSuccessFactor", CLUSTER_HOST_INFO); + Stage stage = new Stage(1, "/tmp", "cluster1", 1L, "testRequestFailureBasedOnSuccessFactor", + CLUSTER_HOST_INFO, "", ""); stage.setStageId(1); stage.addHostRoleExecutionCommand("host1", Role.DATANODE, RoleCommand.UPGRADE, new ServiceComponentHostUpgradeEvent(Role.DATANODE.toString(), "host1", now, "HDP-0.2"), @@ -1310,7 +1315,8 @@ public class TestActionScheduler { private Stage getStageWithSingleTask(String hostname, String clusterName, Role role, RoleCommand roleCommand, Service.Type service, int taskId, int stageId, int requestId) { - Stage stage = new Stage(requestId, "/tmp", clusterName, 1L, "getStageWithSingleTask", CLUSTER_HOST_INFO); + Stage stage = new Stage(requestId, "/tmp", clusterName, 1L, "getStageWithSingleTask", + CLUSTER_HOST_INFO, "{\"host_param\":\"param_value\"}", "{\"stage_param\":\"param_value\"}"); stage.setStageId(stageId); stage.addHostRoleExecutionCommand(hostname, role, roleCommand, new ServiceComponentHostUpgradeEvent(role.toString(), hostname, System.currentTimeMillis(), "HDP-0.2"), @@ -1354,7 +1360,8 @@ public class TestActionScheduler { @Test public void testSuccessFactors() { - Stage s = StageUtils.getATestStage(1, 1, CLUSTER_HOST_INFO); + Stage s = StageUtils.getATestStage(1, 1, CLUSTER_HOST_INFO, + "{\"host_param\":\"param_value\"}", "{\"stage_param\":\"param_value\"}"); assertEquals(new Float(0.5), new Float(s.getSuccessFactor(Role.DATANODE))); assertEquals(new Float(0.5), new Float(s.getSuccessFactor(Role.TASKTRACKER))); assertEquals(new Float(0.5), new Float(s.getSuccessFactor(Role.GANGLIA_MONITOR))); @@ -1427,8 +1434,10 @@ public class TestActionScheduler { ActionDBAccessor db = mock(ActionDBAccessorImpl.class); - Stage s1 = StageUtils.getATestStage(requestId1, stageId, hostname, CLUSTER_HOST_INFO); - Stage s2 = StageUtils.getATestStage(requestId2, stageId, hostname, CLUSTER_HOST_INFO_UPDATED); + Stage s1 = StageUtils.getATestStage(requestId1, stageId, hostname, CLUSTER_HOST_INFO, + "{\"host_param\":\"param_value\"}", "{\"stage_param\":\"param_value\"}"); + Stage s2 = StageUtils.getATestStage(requestId2, stageId, hostname, CLUSTER_HOST_INFO_UPDATED, + "{\"host_param\":\"param_value\"}", "{\"stage_param\":\"param_value\"}"); when(db.getStagesInProgress()).thenReturn(Collections.singletonList(s1)); //Keep large number of attempts so that the task is not expired finally @@ -1499,7 +1508,7 @@ public class TestActionScheduler { final List<Stage> stages = new ArrayList<Stage>(); Stage stage1 = new Stage(1, "/tmp", "cluster1", 1L, "stageWith2Tasks", - CLUSTER_HOST_INFO); + CLUSTER_HOST_INFO, "", ""); addInstallTaskToStage(stage1, hostname1, "cluster1", Role.HBASE_MASTER, RoleCommand.INSTALL, Service.Type.HBASE, 1); addInstallTaskToStage(stage1, hostname1, "cluster1", Role.HBASE_REGIONSERVER, http://git-wip-us.apache.org/repos/asf/ambari/blob/9faeaf5a/ambari-server/src/test/java/org/apache/ambari/server/actionmanager/TestStage.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/actionmanager/TestStage.java b/ambari-server/src/test/java/org/apache/ambari/server/actionmanager/TestStage.java index ba6286f..bde19a1 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/actionmanager/TestStage.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/actionmanager/TestStage.java @@ -34,7 +34,7 @@ public class TestStage { @Test public void testTaskTimeout() { - Stage s = StageUtils.getATestStage(1, 1, "h1", CLUSTER_HOST_INFO); + Stage s = StageUtils.getATestStage(1, 1, "h1", CLUSTER_HOST_INFO, "{\"host_param\":\"param_value\"}", "{\"stage_param\":\"param_value\"}"); s.addHostRoleExecutionCommand("h1", Role.DATANODE, RoleCommand.INSTALL, null, "c1", "HDFS"); s.addHostRoleExecutionCommand("h1", Role.HBASE_MASTER, RoleCommand.INSTALL, @@ -50,7 +50,7 @@ public class TestStage { @Test public void testGetRequestContext() { - Stage stage = new Stage(1, "/logDir", "c1", 1L, "My Context", CLUSTER_HOST_INFO); + Stage stage = new Stage(1, "/logDir", "c1", 1L, "My Context", CLUSTER_HOST_INFO, "", ""); assertEquals("My Context", stage.getRequestContext()); assertEquals(CLUSTER_HOST_INFO, stage.getClusterHostInfo()); } http://git-wip-us.apache.org/repos/asf/ambari/blob/9faeaf5a/ambari-server/src/test/java/org/apache/ambari/server/agent/TestHeartbeatHandler.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/agent/TestHeartbeatHandler.java b/ambari-server/src/test/java/org/apache/ambari/server/agent/TestHeartbeatHandler.java index 5c4a4f1..8c975d6 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/agent/TestHeartbeatHandler.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/agent/TestHeartbeatHandler.java @@ -32,11 +32,16 @@ import static org.apache.ambari.server.agent.DummyHeartbeatConstants.HDFS; import static org.apache.ambari.server.agent.DummyHeartbeatConstants.HDFS_CLIENT; import static org.apache.ambari.server.agent.DummyHeartbeatConstants.NAMENODE; import static org.apache.ambari.server.agent.DummyHeartbeatConstants.SECONDARY_NAMENODE; +import static org.easymock.EasyMock.anyObject; +import static org.easymock.EasyMock.createMockBuilder; +import static org.easymock.EasyMock.createNiceMock; +import static org.easymock.EasyMock.expect; +import static org.easymock.EasyMock.replay; +import static org.easymock.EasyMock.reset; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; -import static org.mockito.Matchers.anyList; import static org.mockito.Matchers.anyString; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -91,7 +96,6 @@ import org.apache.ambari.server.state.svccomphost.ServiceComponentHostStartEvent import org.apache.ambari.server.state.svccomphost.ServiceComponentHostUpgradeEvent; import org.apache.ambari.server.utils.StageUtils; import org.codehaus.jackson.JsonGenerationException; -import static org.easymock.EasyMock.*; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -699,7 +703,8 @@ public class TestHeartbeatHandler { } private void populateActionDB(ActionDBAccessor db, String DummyHostname1) throws AmbariException { - Stage s = new Stage(requestId, "/a/b", DummyCluster, 1L, "heartbeat handler test", "clusterHostInfo"); + Stage s = new Stage(requestId, "/a/b", DummyCluster, 1L, "heartbeat handler test", + "clusterHostInfo", "commandParamsStage", "hostParamsStage"); s.setStageId(stageId); String filename = null; s.addHostRoleExecutionCommand(DummyHostname1, Role.HBASE_MASTER, @@ -1084,7 +1089,7 @@ public class TestHeartbeatHandler { serviceComponentHost1.setState(State.INSTALLING); Stage s = new Stage(1, "/a/b", "cluster1", 1L, "action manager test", - "clusterHostInfo"); + "clusterHostInfo", "commandParamsStage", "hostParamsStage"); s.setStageId(1); s.addHostRoleExecutionCommand(DummyHostname1, Role.DATANODE, RoleCommand.INSTALL, new ServiceComponentHostInstallEvent(Role.DATANODE.toString(), @@ -1657,7 +1662,7 @@ public class TestHeartbeatHandler { serviceComponentHost2.setStackVersion(stack122); Stage s = new Stage(requestId, "/a/b", "cluster1", 1L, "action manager test", - "clusterHostInfo"); + "clusterHostInfo", "commandParamsStage", "hostParamsStage"); s.setStageId(stageId); s.addHostRoleExecutionCommand(DummyHostname1, Role.DATANODE, RoleCommand.UPGRADE, new ServiceComponentHostUpgradeEvent(Role.DATANODE.toString(), http://git-wip-us.apache.org/repos/asf/ambari/blob/9faeaf5a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java index 3aafb9a..ca76dc5 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java @@ -3130,12 +3130,15 @@ public class AmbariManagementControllerTest { Assert.assertNull(stage1.getExecutionCommandWrapper(host2, "DATANODE")); Assert.assertNotNull(stage3.getExecutionCommandWrapper(host1, "HBASE_SERVICE_CHECK")); Assert.assertNotNull(stage2.getExecutionCommandWrapper(host2, "HDFS_SERVICE_CHECK")); + + Type type = new TypeToken<Map<String, String>>() {}.getType(); + for (Stage s : stages) { for (List<ExecutionCommandWrapper> list : s.getExecutionCommands().values()) { for (ExecutionCommandWrapper ecw : list) { if (ecw.getExecutionCommand().getRole().contains("SERVICE_CHECK")) { - Map<String, String> hostParams = ecw.getExecutionCommand().getHostLevelParams(); + Map<String, String> hostParams = StageUtils.getGson().fromJson(s.getHostParamsStage(), type); Assert.assertNotNull(hostParams); Assert.assertTrue(hostParams.size() > 0); Assert.assertTrue(hostParams.containsKey("stack_version")); @@ -3936,12 +3939,14 @@ public class AmbariManagementControllerTest { Assert.assertEquals("a1", task.getRole().name()); Assert.assertEquals("h1", task.getHostName()); ExecutionCommand cmd = task.getExecutionCommandWrapper().getExecutionCommand(); - Map<String, String> commandParameters = cmd.getCommandParams(); + Type type = new TypeToken<Map<String, String>>(){}.getType(); + Map<String, String> hostParametersStage = StageUtils.getGson().fromJson(stage.getHostParamsStage(), type); + Map<String, String> commandParametersStage = StageUtils.getGson().fromJson(stage.getCommandParamsStage(), type); - Assert.assertTrue(commandParameters.containsKey("test")); + Assert.assertTrue(commandParametersStage.containsKey("test")); Assert.assertEquals("HDFS", cmd.getServiceName()); Assert.assertEquals("DATANODE", cmd.getComponentName()); - Assert.assertNotNull(commandParameters.get("jdk_location")); + Assert.assertNotNull(hostParametersStage.get("jdk_location")); resourceFilters.clear(); resourceFilter = new RequestResourceFilter("", "", null); @@ -3965,9 +3970,9 @@ public class AmbariManagementControllerTest { Assert.assertEquals(expectedHosts, actualHosts); cmd = task.getExecutionCommandWrapper().getExecutionCommand(); - commandParameters = cmd.getCommandParams(); + commandParametersStage = StageUtils.getGson().fromJson(stage.getCommandParamsStage(), type); - Assert.assertTrue(commandParameters.containsKey("test")); + Assert.assertTrue(commandParametersStage.containsKey("test")); Assert.assertEquals("HDFS", cmd.getServiceName()); Assert.assertEquals("DATANODE", cmd.getComponentName()); @@ -4063,24 +4068,24 @@ public class AmbariManagementControllerTest { Assert.assertNotNull(stages); HostRoleCommand hrc = null; + Type type = new TypeToken<Map<String, String>>(){}.getType(); for (Stage stage : stages) { for (HostRoleCommand cmd : stage.getOrderedHostRoleCommands()) { if (cmd.getRole().equals(Role.HDFS_CLIENT)) { hrc = cmd; } + Map<String, String> hostParamStage = StageUtils.getGson().fromJson(stage.getHostParamsStage(), type); + Assert.assertTrue(hostParamStage.containsKey(ExecutionCommand.KeyNames.DB_DRIVER_FILENAME)); + Assert.assertTrue(hostParamStage.containsKey(ExecutionCommand.KeyNames.MYSQL_JDBC_URL)); + Assert.assertTrue(hostParamStage.containsKey(ExecutionCommand.KeyNames.ORACLE_JDBC_URL)); } } Assert.assertNotNull(hrc); Assert.assertEquals("RESTART HDFS/HDFS_CLIENT", hrc.getCommandDetail()); Map<String, String> roleParams = hrc.getExecutionCommandWrapper() .getExecutionCommand().getRoleParams(); - Map<String, String> hostParams = hrc.getExecutionCommandWrapper() - .getExecutionCommand().getHostLevelParams(); Assert.assertNotNull(roleParams); - Assert.assertTrue(hostParams.containsKey(ExecutionCommand.KeyNames.DB_DRIVER_FILENAME)); - Assert.assertTrue(hostParams.containsKey(ExecutionCommand.KeyNames.MYSQL_JDBC_URL)); - Assert.assertTrue(hostParams.containsKey(ExecutionCommand.KeyNames.ORACLE_JDBC_URL)); Assert.assertEquals("CLIENT", roleParams.get(ExecutionCommand.KeyNames.COMPONENT_CATEGORY)); Assert.assertTrue(hrc.getExecutionCommandWrapper().getExecutionCommand().getCommandParams().containsKey("hdfs_client")); Assert.assertEquals("abc", hrc.getExecutionCommandWrapper().getExecutionCommand().getCommandParams().get("hdfs_client")); @@ -6321,7 +6326,11 @@ public class AmbariManagementControllerTest { Assert.assertNotNull(nnCommand); ExecutionCommand cmd = nnCommand.getExecutionCommandWrapper().getExecutionCommand(); Assert.assertEquals("a1", cmd.getRole()); - Assert.assertTrue(cmd.getCommandParams().containsKey("test")); + Type type = new TypeToken<Map<String, String>>(){}.getType(); + for (Stage stage : actionDB.getAllStages(response.getRequestId())){ + Map<String, String> commandParamsStage = StageUtils.getGson().fromJson(stage.getCommandParamsStage(), type); + Assert.assertTrue(commandParamsStage.containsKey("test")); + } } @Test @@ -6400,8 +6409,6 @@ public class AmbariManagementControllerTest { int expectedRestartCount = 0; for (HostRoleCommand hrc : storedTasks) { Assert.assertEquals("RESTART", hrc.getCustomCommandName()); - Assert.assertNotNull(hrc.getExecutionCommandWrapper() - .getExecutionCommand().getCommandParams().get("jdk_location")); if (hrc.getHostName().equals("h1") && hrc.getRole().equals(Role.DATANODE)) { expectedRestartCount++; @@ -6506,6 +6513,13 @@ public class AmbariManagementControllerTest { } } } + + Type type = new TypeToken<Map<String, String>>(){}.getType(); + for (Stage stage : actionDB.getAllStages(requestId)){ + Map<String, String> hostParamsStage = StageUtils.getGson().fromJson(stage.getHostParamsStage(), type); + Assert.assertNotNull(hostParamsStage.get("jdk_location")); + } + Assert.assertEquals(true, serviceCheckFound); } @@ -6630,15 +6644,21 @@ public class AmbariManagementControllerTest { .getDesiredState()); List<Stage> stages = actionDB.getAllStages(trackAction.getRequestId()); - Map<String, String> params = stages.get(0).getOrderedHostRoleCommands().get + Type type = new TypeToken<Map<String, String>>(){}.getType(); + + for (Stage stage : stages){ + Map<String, String> params = StageUtils.getGson().fromJson(stage.getHostParamsStage(), type); + Assert.assertEquals("0.1", params.get("stack_version")); + Assert.assertNotNull(params.get("jdk_location")); + Assert.assertNotNull(params.get("db_name")); + Assert.assertNotNull(params.get("mysql_jdbc_url")); + Assert.assertNotNull(params.get("oracle_jdbc_url")); + } + + Map<String, String> paramsCmd = stages.get(0).getOrderedHostRoleCommands().get (0).getExecutionCommandWrapper().getExecutionCommand() .getHostLevelParams(); - Assert.assertEquals("0.1", params.get("stack_version")); - Assert.assertNotNull(params.get("jdk_location")); - Assert.assertNotNull(params.get("repo_info")); - Assert.assertNotNull(params.get("db_name")); - Assert.assertNotNull(params.get("mysql_jdbc_url")); - Assert.assertNotNull(params.get("oracle_jdbc_url")); + Assert.assertNotNull(paramsCmd.get("repo_info")); } @Test @@ -7874,7 +7894,8 @@ public class AmbariManagementControllerTest { List<Stage> stages = new ArrayList<Stage>(); - stages.add(new Stage(requestId1, "/a1", clusterName, 1L, context, CLUSTER_HOST_INFO)); + stages.add(new Stage(requestId1, "/a1", clusterName, 1L, context, + CLUSTER_HOST_INFO, "", "")); stages.get(0).setStageId(1); stages.get(0).addHostRoleExecutionCommand(hostName1, Role.HBASE_MASTER, RoleCommand.START, @@ -7882,14 +7903,16 @@ public class AmbariManagementControllerTest { hostName1, System.currentTimeMillis()), clusterName, "HBASE"); - stages.add(new Stage(requestId1, "/a2", clusterName, 1L, context, CLUSTER_HOST_INFO)); + stages.add(new Stage(requestId1, "/a2", clusterName, 1L, context, + CLUSTER_HOST_INFO, "", "")); stages.get(1).setStageId(2); stages.get(1).addHostRoleExecutionCommand(hostName1, Role.HBASE_CLIENT, RoleCommand.START, new ServiceComponentHostStartEvent(Role.HBASE_CLIENT.toString(), hostName1, System.currentTimeMillis()), clusterName, "HBASE"); - stages.add(new Stage(requestId1, "/a3", clusterName, 1L, context, CLUSTER_HOST_INFO)); + stages.add(new Stage(requestId1, "/a3", clusterName, 1L, context, + CLUSTER_HOST_INFO, "", "")); stages.get(2).setStageId(3); stages.get(2).addHostRoleExecutionCommand(hostName1, Role.HBASE_CLIENT, RoleCommand.START, @@ -7900,14 +7923,16 @@ public class AmbariManagementControllerTest { actionDB.persistActions(request); stages.clear(); - stages.add(new Stage(requestId2, "/a4", clusterName, 1L, context, CLUSTER_HOST_INFO)); + stages.add(new Stage(requestId2, "/a4", clusterName, 1L, context, + CLUSTER_HOST_INFO, "", "")); stages.get(0).setStageId(4); stages.get(0).addHostRoleExecutionCommand(hostName1, Role.HBASE_CLIENT, RoleCommand.START, new ServiceComponentHostStartEvent(Role.HBASE_CLIENT.toString(), hostName1, System.currentTimeMillis()), clusterName, "HBASE"); - stages.add(new Stage(requestId2, "/a5", clusterName, 1L, context, CLUSTER_HOST_INFO)); + stages.add(new Stage(requestId2, "/a5", clusterName, 1L, context, + CLUSTER_HOST_INFO, "", "")); stages.get(1).setStageId(5); stages.get(1).addHostRoleExecutionCommand(hostName1, Role.HBASE_CLIENT, RoleCommand.START, @@ -10142,7 +10167,9 @@ public class AmbariManagementControllerTest { Assert.assertEquals(hostname1, task.getHostName()); ExecutionCommand cmd = task.getExecutionCommandWrapper().getExecutionCommand(); - Assert.assertTrue(cmd.getCommandParams().containsKey("some_custom_param")); + Type type = new TypeToken<Map<String, String>>(){}.getType(); + Map<String, String> commandParamsStage = StageUtils.getGson().fromJson(stage.getCommandParamsStage(), type); + Assert.assertTrue(commandParamsStage.containsKey("some_custom_param")); Assert.assertEquals(null, cmd.getServiceName()); Assert.assertEquals(null, cmd.getComponentName()); @@ -10181,7 +10208,8 @@ public class AmbariManagementControllerTest { Assert.assertEquals(hostname1, task.getHostName()); cmd = task.getExecutionCommandWrapper().getExecutionCommand(); - Assert.assertTrue(cmd.getCommandParams().containsKey("some_custom_param")); + commandParamsStage = StageUtils.getGson().fromJson(stage.getCommandParamsStage(), type); + Assert.assertTrue(commandParamsStage.containsKey("some_custom_param")); Assert.assertEquals(null, cmd.getServiceName()); Assert.assertEquals(null, cmd.getComponentName()); } http://git-wip-us.apache.org/repos/asf/ambari/blob/9faeaf5a/ambari-server/src/test/java/org/apache/ambari/server/stageplanner/TestStagePlanner.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/stageplanner/TestStagePlanner.java b/ambari-server/src/test/java/org/apache/ambari/server/stageplanner/TestStagePlanner.java index cf1e412..f008980 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/stageplanner/TestStagePlanner.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/stageplanner/TestStagePlanner.java @@ -67,7 +67,7 @@ public class TestStagePlanner { RoleGraph rg = new RoleGraph(rco); String hostname = "dummy"; - Stage stage = StageUtils.getATestStage(1, 1, hostname); + Stage stage = StageUtils.getATestStage(1, 1, hostname, "", ""); rg.build(stage); List<Stage> outStages = rg.getStages(); for (Stage s: outStages) { @@ -86,7 +86,7 @@ public class TestStagePlanner { rco.initialize(cluster); RoleGraph rg = new RoleGraph(rco); long now = System.currentTimeMillis(); - Stage stage = StageUtils.getATestStage(1, 1, "host1"); + Stage stage = StageUtils.getATestStage(1, 1, "host1", "", ""); stage.addHostRoleExecutionCommand("host2", Role.HBASE_MASTER, RoleCommand.START, new ServiceComponentHostStartEvent("HBASE_MASTER", "host2", now), "cluster1", "HBASE"); @@ -112,7 +112,7 @@ public class TestStagePlanner { rco.initialize(cluster); RoleGraph rg = new RoleGraph(rco); long now = System.currentTimeMillis(); - Stage stage = StageUtils.getATestStage(1, 1, "host1"); + Stage stage = StageUtils.getATestStage(1, 1, "host1", "", ""); stage.addHostRoleExecutionCommand("host11", Role.SECONDARY_NAMENODE, RoleCommand.START, new ServiceComponentHostStartEvent("SECONDARY_NAMENODE", "host11", now), "cluster1", "HDFS");