AMBARI-21057. Change Storage of Data on Request/Stage/Task To Reduce Redundency (dgrinenko via dlysnichenko)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/f2bbe478 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/f2bbe478 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/f2bbe478 Branch: refs/heads/trunk Commit: f2bbe4781841b871993b3e8dd88e5f56169cf1b8 Parents: f7a1d4e Author: Lisnichenko Dmitro <dlysniche...@hortonworks.com> Authored: Fri May 19 11:52:12 2017 +0300 Committer: Lisnichenko Dmitro <dlysniche...@hortonworks.com> Committed: Fri May 19 11:52:12 2017 +0300 ---------------------------------------------------------------------- .../server/actionmanager/ActionManager.java | 4 +- .../server/actionmanager/ActionScheduler.java | 25 +++-- .../ambari/server/actionmanager/Request.java | 18 +++- .../server/actionmanager/RequestFactory.java | 4 +- .../ambari/server/actionmanager/Stage.java | 14 --- .../server/actionmanager/StageFactory.java | 1 - .../server/actionmanager/StageFactoryImpl.java | 4 +- .../AmbariCustomCommandExecutionHelper.java | 12 ++- .../AmbariManagementControllerImpl.java | 15 ++- .../server/controller/KerberosHelperImpl.java | 51 +++++---- .../ClusterStackVersionResourceProvider.java | 3 +- .../HostStackVersionResourceProvider.java | 4 +- .../internal/RequestResourceProvider.java | 9 +- .../internal/RequestStageContainer.java | 11 +- .../internal/StageResourceProvider.java | 9 +- .../internal/UpgradeResourceProvider.java | 13 +-- .../server/hooks/users/UserHookService.java | 3 +- .../apache/ambari/server/orm/DBAccessor.java | 24 +++++ .../ambari/server/orm/DBAccessorImpl.java | 44 ++++++++ .../server/orm/entities/RequestEntity.java | 19 ++++ .../ambari/server/orm/entities/StageEntity.java | 18 ---- .../server/orm/entities/StageEntity_.java | 4 - .../server/orm/helpers/dbms/DbmsHelper.java | 21 ++++ .../orm/helpers/dbms/GenericDbmsHelper.java | 8 ++ .../server/orm/helpers/dbms/MySqlHelper.java | 12 +++ .../server/orm/helpers/dbms/OracleHelper.java | 12 +++ .../server/orm/helpers/dbms/PostgresHelper.java | 12 +++ .../ambari/server/stageplanner/RoleGraph.java | 2 +- .../server/upgrade/UpgradeCatalog251.java | 29 +++++ .../apache/ambari/server/utils/StageUtils.java | 8 +- .../main/resources/Ambari-DDL-Derby-CREATE.sql | 2 +- .../main/resources/Ambari-DDL-MySQL-CREATE.sql | 2 +- .../main/resources/Ambari-DDL-Oracle-CREATE.sql | 2 +- .../resources/Ambari-DDL-Postgres-CREATE.sql | 2 +- .../resources/Ambari-DDL-SQLAnywhere-CREATE.sql | 2 +- .../resources/Ambari-DDL-SQLServer-CREATE.sql | 2 +- .../src/main/resources/properties.json | 1 + .../ExecutionCommandWrapperTest.java | 4 +- .../ambari/server/actionmanager/StageTest.java | 2 +- .../actionmanager/TestActionDBAccessorImpl.java | 34 +++--- .../server/actionmanager/TestActionManager.java | 8 +- .../actionmanager/TestActionScheduler.java | 43 +++++--- .../ambari/server/actionmanager/TestStage.java | 5 +- .../server/agent/HeartbeatProcessorTest.java | 4 +- .../server/agent/HeartbeatTestHelper.java | 4 +- .../server/agent/TestHeartbeatHandler.java | 4 +- .../AmbariManagementControllerTest.java | 18 ++-- .../server/controller/KerberosHelperTest.java | 105 +++++++++++++++++-- .../internal/CalculatedStatusTest.java | 2 +- ...ClusterStackVersionResourceProviderTest.java | 10 +- .../internal/RequestStageContainerTest.java | 3 +- .../server/hooks/users/UserHookServiceTest.java | 4 +- .../serveraction/ServerActionExecutorTest.java | 9 +- .../server/stageplanner/TestStagePlanner.java | 2 +- .../ambari/server/utils/StageUtilsTest.java | 2 +- 55 files changed, 493 insertions(+), 196 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/f2bbe478/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ActionManager.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ActionManager.java b/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ActionManager.java index 398bc9d..13cdce1 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ActionManager.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ActionManager.java @@ -77,8 +77,8 @@ public class ActionManager { scheduler.stop(); } - public void sendActions(List<Stage> stages, ExecuteActionRequest actionRequest) throws AmbariException { - Request request = requestFactory.createNewFromStages(stages, actionRequest); + public void sendActions(List<Stage> stages, String clusterHostInfo, ExecuteActionRequest actionRequest) throws AmbariException { + Request request = requestFactory.createNewFromStages(stages, clusterHostInfo, actionRequest); sendActions(request, actionRequest); } http://git-wip-us.apache.org/repos/asf/ambari/blob/f2bbe478/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ActionScheduler.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ActionScheduler.java b/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ActionScheduler.java index 316f2bd..d3157e2 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ActionScheduler.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ActionScheduler.java @@ -465,9 +465,10 @@ class ActionScheduler implements Runnable { //Schedule what we have so far + for (ExecutionCommand cmd : commandsToSchedule) { ConfigHelper.processHiddenAttribute(cmd.getConfigurations(), cmd.getConfigurationAttributes(), cmd.getRole(), false); - processHostRole(stage, cmd, commandsToStart, commandsToUpdate); + processHostRole(request, stage, cmd, commandsToStart, commandsToUpdate); } LOG.debug("==> Commands to start: {}", commandsToStart.size()); @@ -1090,7 +1091,7 @@ class ActionScheduler implements Runnable { return serviceEventMap; } - private void processHostRole(Stage s, ExecutionCommand cmd, List<ExecutionCommand> commandsToStart, + private void processHostRole(RequestEntity r, Stage s, ExecutionCommand cmd, List<ExecutionCommand> commandsToStart, List<ExecutionCommand> commandsToUpdate) throws AmbariException { long now = System.currentTimeMillis(); @@ -1106,23 +1107,23 @@ class ActionScheduler implements Runnable { } s.setLastAttemptTime(hostname, roleStr, now); s.incrementAttemptCount(hostname, roleStr); - /** change the hostname in the command for the host itself **/ - cmd.setHostname(hostsMap.getHostMap(hostname)); - //Try to get clusterHostInfo from cache + String requestPK = r.getRequestId().toString(); String stagePk = s.getStageId() + "-" + s.getRequestId(); - Map<String, Set<String>> clusterHostInfo = clusterHostInfoCache.getIfPresent(stagePk); + + // Try to get clusterHostInfo from cache + Map<String, Set<String>> clusterHostInfo = clusterHostInfoCache.getIfPresent(requestPK); if (clusterHostInfo == null) { Type type = new TypeToken<Map<String, Set<String>>>() {}.getType(); - clusterHostInfo = StageUtils.getGson().fromJson(s.getClusterHostInfo(), type); - clusterHostInfoCache.put(stagePk, clusterHostInfo); + clusterHostInfo = StageUtils.getGson().fromJson(r.getClusterHostInfo(), type); + clusterHostInfoCache.put(requestPK, clusterHostInfo); } cmd.setClusterHostInfo(clusterHostInfo); - //Try to get commandParams from cache and merge them with command-level parameters + // Try to get commandParams from cache and merge them with command-level parameters Map<String, String> commandParams = commandParamsStageCache.getIfPresent(stagePk); if (commandParams == null){ @@ -1143,10 +1144,10 @@ class ActionScheduler implements Runnable { } } } catch (ClusterNotFoundException cnfe) { - //NOP + // NOP } - //Try to get hostParams from cache and merge them with command-level parameters + // Try to get hostParams from cache and merge them with command-level parameters Map<String, String> hostParams = hostParamsStageCache.getIfPresent(stagePk); if (hostParams == null) { Type type = new TypeToken<Map<String, String>>() {}.getType(); @@ -1157,6 +1158,8 @@ class ActionScheduler implements Runnable { hostParamsCmd.putAll(hostParams); cmd.setHostLevelParams(hostParamsCmd); + // change the hostname in the command for the host itself + cmd.setHostname(hostsMap.getHostMap(hostname)); commandsToUpdate.add(cmd); } http://git-wip-us.apache.org/repos/asf/ambari/blob/f2bbe478/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/Request.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/Request.java b/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/Request.java index 10e0d57..baf67fe 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/Request.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/Request.java @@ -58,6 +58,7 @@ public class Request { private long createTime; private long startTime; private long endTime; + private String clusterHostInfo; /** * If true, this request can not be executed in parallel with any another @@ -94,6 +95,7 @@ public class Request { this.startTime = -1; this.endTime = -1; this.exclusive = false; + this.clusterHostInfo = "{}"; if (-1L != this.clusterId) { try { @@ -110,7 +112,7 @@ public class Request { * Construct new entity from stages provided */ //TODO remove when not needed - public Request(@Assisted Collection<Stage> stages, Clusters clusters){ + public Request(@Assisted Collection<Stage> stages, @Assisted String clusterHostInfo, Clusters clusters){ if (stages != null && !stages.isEmpty()) { this.stages.addAll(stages); Stage stage = stages.iterator().next(); @@ -129,6 +131,7 @@ public class Request { this.createTime = System.currentTimeMillis(); this.startTime = -1; this.endTime = -1; + this.clusterHostInfo = clusterHostInfo; this.requestType = RequestType.INTERNAL_REQUEST; this.exclusive = false; } else { @@ -143,9 +146,9 @@ public class Request { * Construct new entity from stages provided */ //TODO remove when not needed - public Request(@Assisted Collection<Stage> stages, @Assisted ExecuteActionRequest actionRequest, + public Request(@Assisted Collection<Stage> stages, @Assisted String clusterHostInfo, @Assisted ExecuteActionRequest actionRequest, Clusters clusters, Gson gson) throws AmbariException { - this(stages, clusters); + this(stages, clusterHostInfo, clusters); if (actionRequest != null) { this.resourceFilters = actionRequest.getResourceFilters(); this.operationLevel = actionRequest.getOperationLevel(); @@ -183,6 +186,7 @@ public class Request { this.exclusive = entity.isExclusive(); this.requestContext = entity.getRequestContext(); this.inputs = entity.getInputs(); + this.clusterHostInfo = entity.getClusterHostInfo(); this.requestType = entity.getRequestType(); this.commandName = entity.getCommandName(); @@ -245,6 +249,7 @@ public class Request { requestEntity.setRequestScheduleId(requestScheduleId); requestEntity.setStatus(status); requestEntity.setDisplayStatus(displayStatus); + requestEntity.setClusterHostInfo(clusterHostInfo); //TODO set all fields if (resourceFilters != null) { @@ -281,6 +286,13 @@ public class Request { return requestEntity; } + public String getClusterHostInfo() { + return clusterHostInfo; + } + + public void setClusterHostInfo(String clusterHostInfo) { + this.clusterHostInfo = clusterHostInfo; + } public Long getClusterId() { return Long.valueOf(clusterId); http://git-wip-us.apache.org/repos/asf/ambari/blob/f2bbe478/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/RequestFactory.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/RequestFactory.java b/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/RequestFactory.java index bc0223c..8a22796 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/RequestFactory.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/RequestFactory.java @@ -30,9 +30,9 @@ public interface RequestFactory { Request createNew(long requestId, @Assisted("clusterId") Long clusterName) throws AmbariException; - Request createNewFromStages(Collection<Stage> stages); + Request createNewFromStages(Collection<Stage> stages, String clusterHostInfo); - Request createNewFromStages(Collection<Stage> stages, ExecuteActionRequest actionRequest); + Request createNewFromStages(Collection<Stage> stages, String clusterHostInfo, ExecuteActionRequest actionRequest); Request createExisting(RequestEntity entity); http://git-wip-us.apache.org/repos/asf/ambari/blob/f2bbe478/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/Stage.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/Stage.java b/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/Stage.java index 574afa1..5295536 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/Stage.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/Stage.java @@ -76,7 +76,6 @@ public class Stage { private final String requestContext; private HostRoleStatus status = HostRoleStatus.PENDING; private HostRoleStatus displayStatus = HostRoleStatus.PENDING; - private String clusterHostInfo; private String commandParamsStage; private String hostParamsStage; @@ -110,7 +109,6 @@ public class Stage { @Assisted("clusterName") @Nullable String clusterName, @Assisted("clusterId") long clusterId, @Assisted("requestContext") @Nullable String requestContext, - @Assisted("clusterHostInfo") String clusterHostInfo, @Assisted("commandParamsStage") String commandParamsStage, @Assisted("hostParamsStage") String hostParamsStage, HostRoleCommandFactory hostRoleCommandFactory, ExecutionCommandWrapperFactory ecwFactory) { @@ -120,7 +118,6 @@ public class Stage { this.clusterName = clusterName; this.clusterId = clusterId; this.requestContext = requestContext == null ? "" : requestContext; - this.clusterHostInfo = clusterHostInfo; this.commandParamsStage = commandParamsStage; this.hostParamsStage = hostParamsStage; @@ -155,7 +152,6 @@ public class Stage { } requestContext = stageEntity.getRequestContext(); - clusterHostInfo = stageEntity.getClusterHostInfo(); commandParamsStage = stageEntity.getCommandParamsStage(); hostParamsStage = stageEntity.getHostParamsStage(); commandExecutionType = stageEntity.getCommandExecutionType(); @@ -197,7 +193,6 @@ public class Stage { stageEntity.setRequestContext(requestContext); stageEntity.setHostRoleCommands(new ArrayList<HostRoleCommandEntity>()); stageEntity.setRoleSuccessCriterias(new ArrayList<RoleSuccessCriteriaEntity>()); - stageEntity.setClusterHostInfo(clusterHostInfo); stageEntity.setCommandParamsStage(commandParamsStage); stageEntity.setHostParamsStage(hostParamsStage); stageEntity.setCommandExecutionType(commandExecutionType); @@ -264,14 +259,6 @@ public class Stage { return commandsToScheduleSet; } - public String getClusterHostInfo() { - return clusterHostInfo; - } - - public void setClusterHostInfo(String clusterHostInfo) { - this.clusterHostInfo = clusterHostInfo; - } - public String getCommandParamsStage() { return commandParamsStage; } @@ -935,7 +922,6 @@ public class Stage { builder.append("clusterName=").append(clusterName).append("\n"); builder.append("logDir=").append(logDir).append("\n"); builder.append("requestContext=").append(requestContext).append("\n"); - builder.append("clusterHostInfo=").append(clusterHostInfo).append("\n"); builder.append("commandParamsStage=").append(commandParamsStage).append("\n"); builder.append("hostParamsStage=").append(hostParamsStage).append("\n"); builder.append("status=").append(status).append("\n"); http://git-wip-us.apache.org/repos/asf/ambari/blob/f2bbe478/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/StageFactory.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/StageFactory.java b/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/StageFactory.java index a88558c..0d1a326 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/StageFactory.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/StageFactory.java @@ -29,7 +29,6 @@ public interface StageFactory { @Assisted("clusterName") String clusterName, @Assisted("clusterId") long clusterId, @Assisted("requestContext") String requestContext, - @Assisted("clusterHostInfo") String clusterHostInfo, @Assisted("commandParamsStage") String commandParamsStage, @Assisted("hostParamsStage") String hostParamsStage); http://git-wip-us.apache.org/repos/asf/ambari/blob/f2bbe478/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/StageFactoryImpl.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/StageFactoryImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/StageFactoryImpl.java index 3cad82d..0827639 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/StageFactoryImpl.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/StageFactoryImpl.java @@ -43,7 +43,6 @@ public class StageFactoryImpl implements StageFactory { * @param clusterName Cluster name * @param clusterId Cluster ID * @param requestContext Information about the context of the request - * @param clusterHostInfo Information about the host * @param commandParamsStage Information about the command parameters * @param hostParamsStage Information about the host parameters for the stage * @return An instance of a Stage with the provided params. @@ -54,10 +53,9 @@ public class StageFactoryImpl implements StageFactory { @Assisted("clusterName") String clusterName, @Assisted("clusterId") long clusterId, @Assisted("requestContext") String requestContext, - @Assisted("clusterHostInfo") String clusterHostInfo, @Assisted("commandParamsStage") String commandParamsStage, @Assisted("hostParamsStage") String hostParamsStage) { - return new Stage(requestId, logDir, clusterName, clusterId, requestContext, clusterHostInfo, commandParamsStage, hostParamsStage, + return new Stage(requestId, logDir, clusterName, clusterId, requestContext, commandParamsStage, hostParamsStage, injector.getInstance(HostRoleCommandFactory.class), injector.getInstance(ExecutionCommandWrapperFactory.class)); } http://git-wip-us.apache.org/repos/asf/ambari/blob/f2bbe478/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java index ab8b659..520dcab 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java @@ -75,10 +75,12 @@ import org.apache.ambari.server.controller.spi.Resource; import org.apache.ambari.server.metadata.ActionMetadata; import org.apache.ambari.server.orm.dao.ClusterVersionDAO; import org.apache.ambari.server.orm.dao.HostRoleCommandDAO; +import org.apache.ambari.server.orm.dao.RequestDAO; import org.apache.ambari.server.orm.entities.ClusterVersionEntity; import org.apache.ambari.server.orm.entities.OperatingSystemEntity; import org.apache.ambari.server.orm.entities.RepositoryEntity; import org.apache.ambari.server.orm.entities.RepositoryVersionEntity; +import org.apache.ambari.server.orm.entities.RequestEntity; import org.apache.ambari.server.state.Cluster; import org.apache.ambari.server.state.Clusters; import org.apache.ambari.server.state.CommandScriptDefinition; @@ -176,6 +178,9 @@ public class AmbariCustomCommandExecutionHelper { private ClusterVersionDAO clusterVersionDAO; @Inject + private RequestDAO requestDAO; + + @Inject private HostRoleCommandDAO hostRoleCommandDAO; private Map<String, Map<String, Map<String, String>>> configCredentialsForService = new HashMap<>(); @@ -1014,7 +1019,12 @@ public class AmbariCustomCommandExecutionHelper { StageUtils.getClusterHostInfo(cluster)); // Reset cluster host info as it has changed - stage.setClusterHostInfo(clusterHostInfoJson); + RequestEntity requestEntity = requestDAO.findByPK(stage.getRequestId()); + + if (requestEntity != null) { + requestEntity.setClusterHostInfo(clusterHostInfoJson); + requestDAO.merge(requestEntity); + } Map<String, String> commandParams = new HashMap<>(); if (serviceName.equals(Service.Type.HBASE.name())) { http://git-wip-us.apache.org/repos/asf/ambari/blob/f2bbe478/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java index f9375aa..25b12de 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java @@ -1042,14 +1042,14 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle } private Stage createNewStage(long id, Cluster cluster, long requestId, - String requestContext, String clusterHostInfo, + String requestContext, String commandParamsStage, String hostParamsStage) { String logDir = BASE_LOG_DIR + File.pathSeparator + requestId; Stage stage = stageFactory.createNew(requestId, logDir, null == cluster ? null : cluster.getClusterName(), null == cluster ? -1L : cluster.getClusterId(), - requestContext, clusterHostInfo, commandParamsStage, + requestContext, commandParamsStage, hostParamsStage); stage.setStageId(id); return stage; @@ -2631,8 +2631,7 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle customCommandExecutionHelper.createDefaultHostParams(cluster)); Stage stage = createNewStage(requestStages.getLastStageId(), cluster, - requestStages.getId(), requestProperties.get(REQUEST_CONTEXT_PROPERTY), - clusterHostInfoJson, "{}", hostParamsJson); + requestStages.getId(), requestProperties.get(REQUEST_CONTEXT_PROPERTY),"{}", hostParamsJson); boolean skipFailure = false; if (requestProperties.containsKey(Setting.SETTING_NAME_SKIP_FAILURE) && requestProperties.get(Setting.SETTING_NAME_SKIP_FAILURE).equalsIgnoreCase("true")) { skipFailure = true; @@ -2982,6 +2981,7 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle rg.setCommandExecutionType(CommandExecutionType.DEPENDENCY_ORDERED); } rg.build(stage); + requestStages.setClusterHostInfo(clusterHostInfoJson); requestStages.addStages(rg.getStages()); if (!componentsToEnableKerberos.isEmpty()) { @@ -3067,9 +3067,7 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle Map<String, Set<String>> clusterHostInfo = StageUtils.getClusterHostInfo(cluster); String clusterHostInfoJson = StageUtils.getGson().toJson(clusterHostInfo); Map<String, String> hostParamsCmd = customCommandExecutionHelper.createDefaultHostParams(cluster); - Stage stage = createNewStage(0, cluster, - 1, "", - clusterHostInfoJson, "{}", ""); + Stage stage = createNewStage(0, cluster,1, "","{}", ""); Map<String, Map<String, String>> configTags = configHelper.getEffectiveDesiredTags(cluster, scHost.getHostName()); @@ -4037,7 +4035,7 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle commandParamsForStage = gson.toJson(commandParamsStage); Stage stage = createNewStage(requestStageContainer.getLastStageId(), cluster, requestId, requestContext, - jsons.getClusterHostInfo(), commandParamsForStage, jsons.getHostParamsForStage()); + commandParamsForStage, jsons.getHostParamsForStage()); if (actionRequest.isCommand()) { customCommandExecutionHelper.addExecutionCommandsToStage(actionExecContext, stage, @@ -4058,6 +4056,7 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle List<Stage> stages = rg.getStages(); if (stages != null && !stages.isEmpty()) { + requestStageContainer.setClusterHostInfo(jsons.getClusterHostInfo()); requestStageContainer.addStages(stages); } http://git-wip-us.apache.org/repos/asf/ambari/blob/f2bbe478/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelperImpl.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelperImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelperImpl.java index 6687942..5c4728a 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelperImpl.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelperImpl.java @@ -2186,14 +2186,13 @@ public class KerberosHelperImpl implements KerberosHelper { * @return a newly created Stage */ private Stage createNewStage(long id, Cluster cluster, long requestId, - String requestContext, String clusterHostInfo, - String commandParams, String hostParams) { + String requestContext, String commandParams, String hostParams) { + Stage stage = stageFactory.createNew(requestId, BASE_LOG_DIR + File.pathSeparator + requestId, cluster.getClusterName(), cluster.getClusterId(), requestContext, - clusterHostInfo, commandParams, hostParams); @@ -2221,14 +2220,14 @@ public class KerberosHelperImpl implements KerberosHelper { * @param timeout the timeout for the task/action @return a newly created Stage */ private Stage createServerActionStage(long id, Cluster cluster, long requestId, - String requestContext, String clusterHostInfo, + String requestContext, String commandParams, String hostParams, Class<? extends ServerAction> actionClass, ServiceComponentHostServerActionEvent event, Map<String, String> commandParameters, String commandDetail, Integer timeout) throws AmbariException { - Stage stage = createNewStage(id, cluster, requestId, requestContext, clusterHostInfo, commandParams, hostParams); + Stage stage = createNewStage(id, cluster, requestId, requestContext, commandParams, hostParams); stage.addServerActionCommand(actionClass.getName(), null, Role.AMBARI_SERVER_ACTION, RoleCommand.EXECUTE, cluster.getClusterName(), event, commandParameters, commandDetail, ambariManagementController.findConfigurationTagsWithOverrides(cluster, null), timeout, @@ -2769,7 +2768,6 @@ public class KerberosHelperImpl implements KerberosHelper { cluster, requestStageContainer.getId(), "Preparing Operations", - clusterHostInfoJson, "{}", hostParamsJson, PrepareEnableKerberosServerAction.class, @@ -2780,6 +2778,8 @@ public class KerberosHelperImpl implements KerberosHelper { RoleGraph roleGraph = roleGraphFactory.createNew(roleCommandOrder); roleGraph.build(stage); + + requestStageContainer.setClusterHostInfo(clusterHostInfoJson); requestStageContainer.addStages(roleGraph.getStages()); } @@ -2792,7 +2792,6 @@ public class KerberosHelperImpl implements KerberosHelper { cluster, requestStageContainer.getId(), "Preparing Operations", - clusterHostInfoJson, "{}", hostParamsJson, PrepareKerberosIdentitiesServerAction.class, @@ -2803,6 +2802,8 @@ public class KerberosHelperImpl implements KerberosHelper { RoleGraph roleGraph = roleGraphFactory.createNew(roleCommandOrder); roleGraph.build(stage); + + requestStageContainer.setClusterHostInfo(clusterHostInfoJson); requestStageContainer.addStages(roleGraph.getStages()); } @@ -2815,7 +2816,6 @@ public class KerberosHelperImpl implements KerberosHelper { cluster, requestStageContainer.getId(), "Preparing Operations", - clusterHostInfoJson, "{}", hostParamsJson, PrepareDisableKerberosServerAction.class, @@ -2826,6 +2826,8 @@ public class KerberosHelperImpl implements KerberosHelper { RoleGraph roleGraph = roleGraphFactory.createNew(roleCommandOrder); roleGraph.build(stage); + + requestStageContainer.setClusterHostInfo(clusterHostInfoJson); requestStageContainer.addStages(roleGraph.getStages()); } @@ -2838,7 +2840,6 @@ public class KerberosHelperImpl implements KerberosHelper { cluster, requestStageContainer.getId(), "Create Principals", - clusterHostInfoJson, "{}", hostParamsJson, CreatePrincipalsServerAction.class, @@ -2849,6 +2850,8 @@ public class KerberosHelperImpl implements KerberosHelper { RoleGraph roleGraph = roleGraphFactory.createNew(roleCommandOrder); roleGraph.build(stage); + + requestStageContainer.setClusterHostInfo(clusterHostInfoJson); requestStageContainer.addStages(roleGraph.getStages()); } @@ -2861,7 +2864,6 @@ public class KerberosHelperImpl implements KerberosHelper { cluster, requestStageContainer.getId(), "Destroy Principals", - clusterHostInfoJson, "{}", hostParamsJson, DestroyPrincipalsServerAction.class, @@ -2872,6 +2874,8 @@ public class KerberosHelperImpl implements KerberosHelper { RoleGraph roleGraph = roleGraphFactory.createNew(roleCommandOrder); roleGraph.build(stage); + + requestStageContainer.setClusterHostInfo(clusterHostInfoJson); requestStageContainer.addStages(roleGraph.getStages()); } @@ -2884,7 +2888,6 @@ public class KerberosHelperImpl implements KerberosHelper { cluster, requestStageContainer.getId(), "Configure Ambari Identity", - clusterHostInfoJson, "{}", hostParamsJson, ConfigureAmbariIdentitiesServerAction.class, @@ -2895,6 +2898,8 @@ public class KerberosHelperImpl implements KerberosHelper { RoleGraph roleGraph = roleGraphFactory.createNew(roleCommandOrder); roleGraph.build(stage); + + requestStageContainer.setClusterHostInfo(clusterHostInfoJson); requestStageContainer.addStages(roleGraph.getStages()); } @@ -2907,7 +2912,6 @@ public class KerberosHelperImpl implements KerberosHelper { cluster, requestStageContainer.getId(), "Create Keytabs", - clusterHostInfoJson, "{}", hostParamsJson, CreateKeytabFilesServerAction.class, @@ -2918,6 +2922,8 @@ public class KerberosHelperImpl implements KerberosHelper { RoleGraph roleGraph = roleGraphFactory.createNew(roleCommandOrder); roleGraph.build(stage); + + requestStageContainer.setClusterHostInfo(clusterHostInfoJson); requestStageContainer.addStages(roleGraph.getStages()); } @@ -2933,7 +2939,6 @@ public class KerberosHelperImpl implements KerberosHelper { cluster, requestStageContainer.getId(), "Distribute Keytabs", - clusterHostInfoJson, StageUtils.getGson().toJson(commandParameters), hostParamsJson); @@ -2958,6 +2963,8 @@ public class KerberosHelperImpl implements KerberosHelper { RoleGraph roleGraph = roleGraphFactory.createNew(roleCommandOrder); roleGraph.build(stage); + + requestStageContainer.setClusterHostInfo(clusterHostInfoJson); requestStageContainer.addStages(roleGraph.getStages()); } @@ -2999,12 +3006,13 @@ public class KerberosHelperImpl implements KerberosHelper { cluster, requestStageContainer.getId(), "Disable security", - clusterHostInfoJson, StageUtils.getGson().toJson(commandParameters), hostParamsJson); addDisableSecurityCommandToAllServices(cluster, stage); RoleGraph roleGraph = roleGraphFactory.createNew(roleCommandOrder); roleGraph.build(stage); + + requestStageContainer.setClusterHostInfo(clusterHostInfoJson); requestStageContainer.addStages(roleGraph.getStages()); } @@ -3042,7 +3050,6 @@ public class KerberosHelperImpl implements KerberosHelper { cluster, requestStageContainer.getId(), "Stopping ZooKeeper", - clusterHostInfoJson, StageUtils.getGson().toJson(commandParameters), hostParamsJson); for (ServiceComponent component : zookeeper.getServiceComponents().values()) { @@ -3056,6 +3063,8 @@ public class KerberosHelperImpl implements KerberosHelper { } RoleGraph roleGraph = roleGraphFactory.createNew(roleCommandOrder); roleGraph.build(stage); + + requestStageContainer.setClusterHostInfo(clusterHostInfoJson); requestStageContainer.addStages(roleGraph.getStages()); } @@ -3071,7 +3080,6 @@ public class KerberosHelperImpl implements KerberosHelper { cluster, requestStageContainer.getId(), "Delete Keytabs", - clusterHostInfoJson, StageUtils.getGson().toJson(commandParameters), hostParamsJson); @@ -3099,6 +3107,8 @@ public class KerberosHelperImpl implements KerberosHelper { RoleGraph roleGraph = roleGraphFactory.createNew(roleCommandOrder); roleGraph.build(stage); + + requestStageContainer.setClusterHostInfo(clusterHostInfoJson); requestStageContainer.addStages(roleGraph.getStages()); } @@ -3111,7 +3121,6 @@ public class KerberosHelperImpl implements KerberosHelper { cluster, requestStageContainer.getId(), "Update Configurations", - clusterHostInfoJson, "{}", hostParamsJson, UpdateKerberosConfigsServerAction.class, @@ -3122,6 +3131,8 @@ public class KerberosHelperImpl implements KerberosHelper { RoleGraph roleGraph = roleGraphFactory.createNew(roleCommandOrder); roleGraph.build(stage); + + requestStageContainer.setClusterHostInfo(clusterHostInfoJson); requestStageContainer.addStages(roleGraph.getStages()); } @@ -3145,7 +3156,6 @@ public class KerberosHelperImpl implements KerberosHelper { cluster, requestStageContainer.getId(), "Finalize Operations", - clusterHostInfoJson, "{}", hostParamsJson, FinalizeKerberosServerAction.class, @@ -3155,6 +3165,8 @@ public class KerberosHelperImpl implements KerberosHelper { RoleGraph roleGraph = roleGraphFactory.createNew(roleCommandOrder); roleGraph.build(stage); + + requestStageContainer.setClusterHostInfo(clusterHostInfoJson); requestStageContainer.addStages(roleGraph.getStages()); } @@ -3167,7 +3179,6 @@ public class KerberosHelperImpl implements KerberosHelper { cluster, requestStageContainer.getId(), "Kerberization Clean Up", - clusterHostInfoJson, "{}", hostParamsJson, CleanupServerAction.class, @@ -3178,6 +3189,8 @@ public class KerberosHelperImpl implements KerberosHelper { RoleGraph roleGraph = roleGraphFactory.createNew(roleCommandOrder); roleGraph.build(stage); + + requestStageContainer.setClusterHostInfo(clusterHostInfoJson); requestStageContainer.addStages(roleGraph.getStages()); } } http://git-wip-us.apache.org/repos/asf/ambari/blob/f2bbe478/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterStackVersionResourceProvider.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterStackVersionResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterStackVersionResourceProvider.java index a8bb696..f8016a5 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterStackVersionResourceProvider.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterStackVersionResourceProvider.java @@ -542,7 +542,7 @@ public class ClusterStackVersionResourceProvider extends AbstractControllerResou } Stage stage = stageFactory.createNew(req.getId(), "/tmp/ambari", cluster.getClusterName(), - cluster.getClusterId(), stageName, clusterHostInfoJson, "{}", hostParamsJson); + cluster.getClusterId(), stageName, "{}", hostParamsJson); // if you have 1000 hosts (10 stages with 100 installs), we want to ensure // that a single failure doesn't cause all other stages to abort; set the @@ -590,6 +590,7 @@ public class ClusterStackVersionResourceProvider extends AbstractControllerResou repoVersionEnt.getDisplayName())); } + req.setClusterHostInfo(clusterHostInfoJson); req.addStages(stages); req.persist(); http://git-wip-us.apache.org/repos/asf/ambari/blob/f2bbe478/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/HostStackVersionResourceProvider.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/HostStackVersionResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/HostStackVersionResourceProvider.java index 811ce9b..92edeb8 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/HostStackVersionResourceProvider.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/HostStackVersionResourceProvider.java @@ -513,7 +513,6 @@ public class HostStackVersionResourceProvider extends AbstractControllerResource cluster.getClusterName(), cluster.getClusterId(), caption, - clusterHostInfoJson, "{}", StageUtils.getGson().toJson(hostLevelParams)); @@ -522,6 +521,7 @@ public class HostStackVersionResourceProvider extends AbstractControllerResource stageId = 1L; } stage.setStageId(stageId); + req.setClusterHostInfo(clusterHostInfoJson); req.addStages(Collections.singletonList(stage)); try { @@ -561,7 +561,6 @@ public class HostStackVersionResourceProvider extends AbstractControllerResource cluster.getClusterName(), cluster.getClusterId(), caption, - clusterHostInfoJson, StageUtils.getGson().toJson(commandParams), StageUtils.getGson().toJson(hostLevelParams)); @@ -570,6 +569,7 @@ public class HostStackVersionResourceProvider extends AbstractControllerResource stageId = 1L; } stage.setStageId(stageId); + req.setClusterHostInfo(clusterHostInfoJson); req.addStages(Collections.singletonList(stage)); actionContext = new ActionExecutionContext( http://git-wip-us.apache.org/repos/asf/ambari/blob/f2bbe478/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/RequestResourceProvider.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/RequestResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/RequestResourceProvider.java index c405995..d82ff25 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/RequestResourceProvider.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/RequestResourceProvider.java @@ -103,6 +103,7 @@ public class RequestResourceProvider extends AbstractControllerResourceProvider public static final String REQUEST_SOURCE_SCHEDULE_HREF = "Requests/request_schedule/href"; protected static final String REQUEST_TYPE_ID = "Requests/type"; protected static final String REQUEST_INPUTS_ID = "Requests/inputs"; + protected static final String REQUEST_CLUSTER_HOST_INFO_ID = "Requests/cluster_host_info"; protected static final String REQUEST_RESOURCE_FILTER_ID = "Requests/resource_filters"; protected static final String REQUEST_OPERATION_LEVEL_ID = "Requests/operation_level"; protected static final String REQUEST_CREATE_TIME_ID = "Requests/create_time"; @@ -158,7 +159,9 @@ public class RequestResourceProvider extends AbstractControllerResourceProvider REQUEST_QUEUED_TASK_CNT_ID, REQUEST_PROGRESS_PERCENT_ID, REQUEST_REMOVE_PENDING_HOST_REQUESTS_ID, - REQUEST_PENDING_HOST_REQUEST_COUNT_ID); + REQUEST_PENDING_HOST_REQUEST_COUNT_ID, + REQUEST_CLUSTER_HOST_INFO_ID + ); // ----- Constructors ---------------------------------------------------- @@ -752,6 +755,10 @@ public class RequestResourceProvider extends AbstractControllerResourceProvider resource.setProperty(REQUEST_INPUTS_ID, value); } + if (isPropertyRequested(REQUEST_CLUSTER_HOST_INFO_ID, requestedPropertyIds)) { + resource.setProperty(REQUEST_CLUSTER_HOST_INFO_ID, entity.getClusterHostInfo()); + } + setResourceProperty(resource, REQUEST_RESOURCE_FILTER_ID, org.apache.ambari.server.actionmanager.Request.filtersFromEntity(entity), requestedPropertyIds); http://git-wip-us.apache.org/repos/asf/ambari/blob/f2bbe478/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/RequestStageContainer.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/RequestStageContainer.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/RequestStageContainer.java index c37be91..3f67704 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/RequestStageContainer.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/RequestStageContainer.java @@ -65,6 +65,8 @@ public class RequestStageContainer { private ExecuteActionRequest actionRequest = null; + private String clusterHostInfo = null; + /** * Logger */ @@ -99,6 +101,7 @@ public class RequestStageContainer { this.requestFactory = factory; this.actionManager = manager; this.actionRequest = actionRequest; + this.clusterHostInfo = "{}"; } /** @@ -110,6 +113,10 @@ public class RequestStageContainer { return id; } + public void setClusterHostInfo(String clusterHostInfo){ + this.clusterHostInfo = clusterHostInfo; + } + /** * Add stages to request. * @@ -202,8 +209,8 @@ public class RequestStageContainer { public void persist() throws AmbariException { if (!stages.isEmpty()) { Request request = (null == actionRequest) - ? requestFactory.createNewFromStages(stages) - : requestFactory.createNewFromStages(stages, actionRequest); + ? requestFactory.createNewFromStages(stages, clusterHostInfo) + : requestFactory.createNewFromStages(stages, clusterHostInfo, actionRequest); if (null != requestContext) { request.setRequestContext(requestContext); http://git-wip-us.apache.org/repos/asf/ambari/blob/f2bbe478/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StageResourceProvider.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StageResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StageResourceProvider.java index ec3688d..06aa68b 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StageResourceProvider.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StageResourceProvider.java @@ -47,6 +47,7 @@ import org.apache.ambari.server.controller.spi.UnsupportedPropertyException; import org.apache.ambari.server.controller.utilities.PredicateHelper; import org.apache.ambari.server.orm.dao.HostRoleCommandDAO; import org.apache.ambari.server.orm.dao.HostRoleCommandStatusSummaryDTO; +import org.apache.ambari.server.orm.dao.RequestDAO; import org.apache.ambari.server.orm.dao.StageDAO; import org.apache.ambari.server.orm.entities.StageEntity; import org.apache.ambari.server.state.Cluster; @@ -91,7 +92,6 @@ public class StageResourceProvider extends AbstractControllerResourceProvider im public static final String STAGE_REQUEST_ID = "Stage/request_id"; public static final String STAGE_LOG_INFO = "Stage/log_info"; public static final String STAGE_CONTEXT = "Stage/context"; - public static final String STAGE_CLUSTER_HOST_INFO = "Stage/cluster_host_info"; public static final String STAGE_COMMAND_PARAMS = "Stage/command_params"; public static final String STAGE_HOST_PARAMS = "Stage/host_params"; public static final String STAGE_SKIPPABLE = "Stage/skippable"; @@ -119,7 +119,6 @@ public class StageResourceProvider extends AbstractControllerResourceProvider im PROPERTY_IDS.add(STAGE_REQUEST_ID); PROPERTY_IDS.add(STAGE_LOG_INFO); PROPERTY_IDS.add(STAGE_CONTEXT); - PROPERTY_IDS.add(STAGE_CLUSTER_HOST_INFO); PROPERTY_IDS.add(STAGE_COMMAND_PARAMS); PROPERTY_IDS.add(STAGE_HOST_PARAMS); PROPERTY_IDS.add(STAGE_SKIPPABLE); @@ -307,12 +306,6 @@ public class StageResourceProvider extends AbstractControllerResourceProvider im setResourceProperty(resource, STAGE_REQUEST_ID, entity.getRequestId(), requestedIds); setResourceProperty(resource, STAGE_CONTEXT, entity.getRequestContext(), requestedIds); - // this property is lazy loaded in JPA; don't use it unless requested - if (isPropertyRequested(STAGE_CLUSTER_HOST_INFO, requestedIds)) { - resource.setProperty(STAGE_CLUSTER_HOST_INFO, entity.getClusterHostInfo()); - } - - // this property is lazy loaded in JPA; don't use it unless requested if (isPropertyRequested(STAGE_COMMAND_PARAMS, requestedIds)) { String value = entity.getCommandParamsStage(); if (!StringUtils.isBlank(value)) { http://git-wip-us.apache.org/repos/asf/ambari/blob/f2bbe478/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java index 623851a..0ebf3aa 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java @@ -1371,8 +1371,7 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider cluster, context.getEffectiveStackId()); Stage stage = s_stageFactory.get().createNew(request.getId().longValue(), "/tmp/ambari", - cluster.getClusterName(), cluster.getClusterId(), entity.getText(), - jsons.getClusterHostInfo(), jsons.getCommandParamsForStage(), + cluster.getClusterName(), cluster.getClusterId(), entity.getText(), jsons.getCommandParamsForStage(), jsons.getHostParamsForStage()); stage.setSkippable(skippable); @@ -1454,8 +1453,7 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider cluster, context.getEffectiveStackId()); Stage stage = s_stageFactory.get().createNew(request.getId().longValue(), "/tmp/ambari", - cluster.getClusterName(), cluster.getClusterId(), entity.getText(), - jsons.getClusterHostInfo(), jsons.getCommandParamsForStage(), + cluster.getClusterName(), cluster.getClusterId(), entity.getText(), jsons.getCommandParamsForStage(), jsons.getHostParamsForStage()); stage.setSkippable(skippable); @@ -1516,8 +1514,7 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider cluster, context.getEffectiveStackId()); Stage stage = s_stageFactory.get().createNew(request.getId().longValue(), "/tmp/ambari", - cluster.getClusterName(), cluster.getClusterId(), entity.getText(), - jsons.getClusterHostInfo(), jsons.getCommandParamsForStage(), + cluster.getClusterName(), cluster.getClusterId(), entity.getText(), jsons.getCommandParamsForStage(), jsons.getHostParamsForStage()); stage.setSkippable(skippable); @@ -1648,8 +1645,8 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider cluster, context.getEffectiveStackId()); Stage stage = s_stageFactory.get().createNew(request.getId().longValue(), "/tmp/ambari", - cluster.getClusterName(), cluster.getClusterId(), stageText, jsons.getClusterHostInfo(), - jsons.getCommandParamsForStage(), jsons.getHostParamsForStage()); + cluster.getClusterName(), cluster.getClusterId(), stageText, jsons.getCommandParamsForStage(), + jsons.getHostParamsForStage()); stage.setSkippable(skippable); stage.setAutoSkipFailureSupported(supportsAutoSkipOnFailure); http://git-wip-us.apache.org/repos/asf/ambari/blob/f2bbe478/ambari-server/src/main/java/org/apache/ambari/server/hooks/users/UserHookService.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/hooks/users/UserHookService.java b/ambari-server/src/main/java/org/apache/ambari/server/hooks/users/UserHookService.java index 69463ab..149e2f0 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/hooks/users/UserHookService.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/hooks/users/UserHookService.java @@ -139,7 +139,7 @@ public class UserHookService implements HookService { String stageContextText = String.format(POST_USER_CREATION_REQUEST_CONTEXT, ctx.getUserGroups().size()); Stage stage = stageFactory.createNew(requestStageContainer.getId(), configuration.getServerTempDir() + File.pathSeparatorChar + requestStageContainer.getId(), clsData.getClusterName(), - clsData.getClusterId(), stageContextText, "{}", "{}", "{}"); + clsData.getClusterId(), stageContextText, "{}", "{}"); stage.setStageId(requestStageContainer.getLastStageId()); ServiceComponentHostServerActionEvent serverActionEvent = new ServiceComponentHostServerActionEvent("ambari-server-host", System.currentTimeMillis()); @@ -148,6 +148,7 @@ public class UserHookService implements HookService { stage.addServerActionCommand(PostUserCreationHookServerAction.class.getName(), "ambari", Role.AMBARI_SERVER_ACTION, RoleCommand.EXECUTE, clsData.getClusterName(), serverActionEvent, commandParams, stageContextText, null, null, false, false); + requestStageContainer.setClusterHostInfo("{}"); requestStageContainer.addStages(Collections.singletonList(stage)); requestStageContainer.persist(); http://git-wip-us.apache.org/repos/asf/ambari/blob/f2bbe478/ambari-server/src/main/java/org/apache/ambari/server/orm/DBAccessor.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/orm/DBAccessor.java b/ambari-server/src/main/java/org/apache/ambari/server/orm/DBAccessor.java index bf8ff48..4f29d61 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/orm/DBAccessor.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/orm/DBAccessor.java @@ -622,6 +622,30 @@ public interface DBAccessor { */ void addDefaultConstraint(String tableName, DBColumnInfo column) throws SQLException; + /** + * Move column data from {@code sourceTableName} to {@code targetTableName} using {@code sourceIDFieldName} and + * {@code targetIDFieldName} keys to match right rows + * + * @param sourceTableName + * the source table name + * @param sourceColumn + * the source column name + * @param sourceIDFieldName + * the source id key filed name matched with {@code targetIDFieldName} + * @param targetTableName + * the target table name + * @param targetColumn + * the target column name + * @param targetIDFieldName + * the target id key name matched with {@code sourceIDFieldName} + * @param isColumnNullable + * should be target column nullable or not + * + * @throws SQLException + */ + void moveColumnToAnotherTable(String sourceTableName, DBColumnInfo sourceColumn, String sourceIDFieldName, + String targetTableName, DBColumnInfo targetColumn, String targetIDFieldName, boolean isColumnNullable) throws SQLException; + enum DbType { ORACLE, MYSQL, http://git-wip-us.apache.org/repos/asf/ambari/blob/f2bbe478/ambari-server/src/main/java/org/apache/ambari/server/orm/DBAccessorImpl.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/orm/DBAccessorImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/orm/DBAccessorImpl.java index c11589d..9c6425c 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/orm/DBAccessorImpl.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/orm/DBAccessorImpl.java @@ -1304,4 +1304,48 @@ public class DBAccessorImpl implements DBAccessor { return valueString; } + + /** + * Move column data from {@code sourceTableName} to {@code targetTableName} using {@code sourceIDFieldName} and + * {@code targetIDFieldName} keys to match right rows + * + * @param sourceTableName + * the source table name + * @param sourceColumn + * the source column name + * @param sourceIDFieldName + * the source id key filed name matched with {@code targetIDFieldName} + * @param targetTableName + * the target table name + * @param targetColumn + * the target column name + * @param targetIDFieldName + * the target id key name matched with {@code sourceIDFieldName} + * @param isColumnNullable + * should be target column nullable or not + * + * @throws SQLException + */ + @Override + public void moveColumnToAnotherTable(String sourceTableName, DBColumnInfo sourceColumn, String sourceIDFieldName, + String targetTableName, DBColumnInfo targetColumn, String targetIDFieldName, boolean isColumnNullable) throws SQLException { + + if (this.tableHasColumn(sourceTableName, sourceIDFieldName)) { + + final String moveSQL = dbmsHelper.getCopyColumnToAnotherTableStatement(sourceTableName, sourceColumn.getName(), + sourceIDFieldName, targetTableName, targetColumn.getName(),targetIDFieldName); + + targetColumn.setNullable(true); // setting column nullable by default + + this.addColumn(targetTableName, targetColumn); + this.executeUpdate(moveSQL, false); + + if (!isColumnNullable) { + // this can will trigger exception if some record is null + // ToDo: add default option + this.setColumnNullable(targetTableName, targetColumn.getName(), false); + } + this.dropColumn(sourceTableName, sourceColumn.getName()); + } + } } http://git-wip-us.apache.org/repos/asf/ambari/blob/f2bbe478/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/RequestEntity.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/RequestEntity.java b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/RequestEntity.java index 099d08f..adf6647 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/RequestEntity.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/RequestEntity.java @@ -26,6 +26,7 @@ import javax.persistence.Column; import javax.persistence.Entity; import javax.persistence.EnumType; import javax.persistence.Enumerated; +import javax.persistence.FetchType; import javax.persistence.Id; import javax.persistence.JoinColumn; import javax.persistence.Lob; @@ -67,6 +68,16 @@ public class RequestEntity { @Basic private String commandName; + /** + * On large clusters, this value can be in the 10,000's of kilobytes. During + * an upgrade, all stages are loaded in memory for every request, which can + * lead to an OOM. As a result, lazy load this since it's barely ever + * requested or used. + */ + @Column(name = "cluster_host_info") + @Basic(fetch = FetchType.LAZY) + private byte[] clusterHostInfo; + @Column(name = "inputs") @Lob private byte[] inputs = new byte[0]; @@ -151,6 +162,14 @@ public class RequestEntity { this.stages = stages; } + public String getClusterHostInfo() { + return clusterHostInfo == null ? "{}" : new String(clusterHostInfo); + } + + public void setClusterHostInfo(String clusterHostInfo) { + this.clusterHostInfo = clusterHostInfo.getBytes(); + } + public Long getCreateTime() { return createTime; } http://git-wip-us.apache.org/repos/asf/ambari/blob/f2bbe478/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/StageEntity.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/StageEntity.java b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/StageEntity.java index f688412..6ee0a3b 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/StageEntity.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/StageEntity.java @@ -96,16 +96,6 @@ public class StageEntity { * lead to an OOM. As a result, lazy load this since it's barely ever * requested or used. */ - @Column(name = "cluster_host_info") - @Basic(fetch = FetchType.LAZY) - private byte[] clusterHostInfo; - - /** - * On large clusters, this value can be in the 10,000's of kilobytes. During - * an upgrade, all stages are loaded in memory for every request, which can - * lead to an OOM. As a result, lazy load this since it's barely ever - * requested or used. - */ @Column(name = "command_params") @Basic(fetch = FetchType.LAZY) private byte[] commandParamsStage; @@ -187,14 +177,6 @@ public class StageEntity { return defaultString(requestContext); } - public String getClusterHostInfo() { - return clusterHostInfo == null ? new String() : new String(clusterHostInfo); - } - - public void setClusterHostInfo(String clusterHostInfo) { - this.clusterHostInfo = clusterHostInfo.getBytes(); - } - public String getCommandParamsStage() { return commandParamsStage == null ? new String() : new String(commandParamsStage); } http://git-wip-us.apache.org/repos/asf/ambari/blob/f2bbe478/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/StageEntity_.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/StageEntity_.java b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/StageEntity_.java index dc39e55..637a18b 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/StageEntity_.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/StageEntity_.java @@ -41,7 +41,6 @@ public class StageEntity_ { public static volatile SingularAttribute<StageEntity, String> logInfo; public static volatile SingularAttribute<StageEntity, String> requestContext; - public static volatile SingularAttribute<StageEntity, byte[]> clusterHostInfo; public static volatile SingularAttribute<StageEntity, byte[]> commandParamsStage; public static volatile SingularAttribute<StageEntity, byte[]> hostParamsStage; @@ -74,9 +73,6 @@ public class StageEntity_ { mapping.put(StageResourceProvider.STAGE_CONTEXT, Collections.singletonList(requestContext)); - mapping.put(StageResourceProvider.STAGE_CLUSTER_HOST_INFO, - Collections.singletonList(clusterHostInfo)); - mapping.put(StageResourceProvider.STAGE_COMMAND_PARAMS, Collections.singletonList(commandParamsStage)); http://git-wip-us.apache.org/repos/asf/ambari/blob/f2bbe478/ambari-server/src/main/java/org/apache/ambari/server/orm/helpers/dbms/DbmsHelper.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/orm/helpers/dbms/DbmsHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/orm/helpers/dbms/DbmsHelper.java index d374ddc..7f74bb0 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/orm/helpers/dbms/DbmsHelper.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/orm/helpers/dbms/DbmsHelper.java @@ -127,6 +127,27 @@ public interface DbmsHelper { String getSetNullableStatement(String tableName, DBAccessor.DBColumnInfo columnInfo, boolean nullable); /** + * Get's the {@code UPDATE} statement for {@code sourceTable} for copy column from {@code targetTable} by matching + * table keys {@code sourceIDColumnName} and {@code targetIDColumnName} + * + * @param sourceTable + * the source table name + * @param sourceColumnName + * the source column name + * @param sourceIDColumnName + * source key id column which would be used to math right rows for {@code targetTable} + * @param targetTable + * the destination table name + * @param targetColumnName + * the destination column name + * @param targetIDColumnName + * destination key id column name which should math {@code sourceIDColumnName} + * @return + */ + String getCopyColumnToAnotherTableStatement(String sourceTable, String sourceColumnName, String sourceIDColumnName, + String targetTable, String targetColumnName, String targetIDColumnName); + + /** * Gets whether the database platform supports adding contraints after the * {@code NULL} constraint. Some database, such as Oracle, don't allow this. * Unfortunately, EclipsLink hard codes the order of constraints. http://git-wip-us.apache.org/repos/asf/ambari/blob/f2bbe478/ambari-server/src/main/java/org/apache/ambari/server/orm/helpers/dbms/GenericDbmsHelper.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/orm/helpers/dbms/GenericDbmsHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/orm/helpers/dbms/GenericDbmsHelper.java index f60c138..7e3092d 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/orm/helpers/dbms/GenericDbmsHelper.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/orm/helpers/dbms/GenericDbmsHelper.java @@ -78,6 +78,14 @@ public class GenericDbmsHelper implements DbmsHelper { return stringBuilder.toString(); } + /** + {@inheritDoc} + */ + @Override + public String getCopyColumnToAnotherTableStatement(String sourceTable, String sourceColumnName, String sourceIDColumnName, String targetTable, String targetColumnName, String targetIDColumnName) { + throw new UnsupportedOperationException("Column copy is not supported for generic DB"); + } + public StringBuilder writeAlterTableClause(StringBuilder builder, String tableName) { builder.append("ALTER TABLE ").append(tableName).append(" "); return builder; http://git-wip-us.apache.org/repos/asf/ambari/blob/f2bbe478/ambari-server/src/main/java/org/apache/ambari/server/orm/helpers/dbms/MySqlHelper.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/orm/helpers/dbms/MySqlHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/orm/helpers/dbms/MySqlHelper.java index c693be5..0daea72 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/orm/helpers/dbms/MySqlHelper.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/orm/helpers/dbms/MySqlHelper.java @@ -93,4 +93,16 @@ public class MySqlHelper extends GenericDbmsHelper { } return defaultWriter; } + + /** + {@inheritDoc} + */ + @Override + public String getCopyColumnToAnotherTableStatement(String sourceTable, String sourceColumnName, + String sourceIDColumnName, String targetTable, String targetColumnName, String targetIDColumnName) { + + return String.format("UPDATE %1$s AS a INNER JOIN %2$s AS b ON a.%5$s = b.%6$s SET a.%3$s = b.%4$s", + targetTable, sourceTable, targetColumnName, sourceColumnName, targetIDColumnName, sourceIDColumnName); + } + } http://git-wip-us.apache.org/repos/asf/ambari/blob/f2bbe478/ambari-server/src/main/java/org/apache/ambari/server/orm/helpers/dbms/OracleHelper.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/orm/helpers/dbms/OracleHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/orm/helpers/dbms/OracleHelper.java index b5955b4..73356d1 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/orm/helpers/dbms/OracleHelper.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/orm/helpers/dbms/OracleHelper.java @@ -79,4 +79,16 @@ public class OracleHelper extends GenericDbmsHelper { public boolean isConstraintSupportedAfterNullability() { return false; } + + /** + {@inheritDoc} + */ + @Override + public String getCopyColumnToAnotherTableStatement(String sourceTable, String sourceColumnName, + String sourceIDColumnName, String targetTable, String targetColumnName, String targetIDColumnName) { + + // sub-query should return only one value, ROWNUM is safe-guard for this + return String.format("UPDATE %1$s a SET (a.%3$s) = (SELECT b.%4$s FROM %2$s b WHERE b.%6$s = a.%5$s and ROWNUM < 2)", + targetTable, sourceTable, targetColumnName, sourceColumnName, targetIDColumnName, sourceIDColumnName); + } } http://git-wip-us.apache.org/repos/asf/ambari/blob/f2bbe478/ambari-server/src/main/java/org/apache/ambari/server/orm/helpers/dbms/PostgresHelper.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/orm/helpers/dbms/PostgresHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/orm/helpers/dbms/PostgresHelper.java index 2237f86..37c1184 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/orm/helpers/dbms/PostgresHelper.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/orm/helpers/dbms/PostgresHelper.java @@ -44,6 +44,18 @@ public class PostgresHelper extends GenericDbmsHelper { return builder; } + /** + {@inheritDoc} + */ + @Override + public String getCopyColumnToAnotherTableStatement(String sourceTable, String sourceColumnName, + String sourceIDColumnName, String targetTable, String targetColumnName, String targetIDColumnName) { + + return String.format("UPDATE %1$s AS a SET %3$s = b.%4$s FROM %2$s AS b WHERE a.%5$s = b.%6$s", + targetTable, sourceTable, targetColumnName, sourceColumnName, targetIDColumnName, sourceIDColumnName); + } + + @Override public StringBuilder writeSetNullableString(StringBuilder builder, String tableName, DBAccessor.DBColumnInfo columnInfo, boolean nullable) { http://git-wip-us.apache.org/repos/asf/ambari/blob/f2bbe478/ambari-server/src/main/java/org/apache/ambari/server/stageplanner/RoleGraph.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/stageplanner/RoleGraph.java b/ambari-server/src/main/java/org/apache/ambari/server/stageplanner/RoleGraph.java index b54c7c7..b6b756b 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/stageplanner/RoleGraph.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/stageplanner/RoleGraph.java @@ -283,7 +283,7 @@ public class RoleGraph { Stage newStage = stageFactory.createNew(origStage.getRequestId(), origStage.getLogDir(), origStage.getClusterName(), origStage.getClusterId(), - origStage.getRequestContext(), origStage.getClusterHostInfo(), + origStage.getRequestContext(), origStage.getCommandParamsStage(), origStage.getHostParamsStage()); newStage.setSuccessFactors(origStage.getSuccessFactors()); newStage.setSkippable(origStage.isSkippable()); http://git-wip-us.apache.org/repos/asf/ambari/blob/f2bbe478/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog251.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog251.java b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog251.java index 745890c..5ed33a8 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog251.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog251.java @@ -33,6 +33,8 @@ import org.apache.commons.lang.StringUtils; import com.google.inject.Inject; import com.google.inject.Injector; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * The {@link UpgradeCatalog251} upgrades Ambari from 2.5.0 to 2.5.1. @@ -44,6 +46,17 @@ public class UpgradeCatalog251 extends AbstractUpgradeCatalog { protected static final String KAFKA_BROKER_CONFIG = "kafka-broker"; + private static final String STAGE_TABLE = "stage"; + private static final String REQUEST_TABLE = "request"; + private static final String CLUSTER_HOST_INFO_COLUMN = "cluster_host_info"; + private static final String REQUEST_ID_COLUMN = "request_id"; + + + /** + * Logger. + */ + private static final Logger LOG = LoggerFactory.getLogger(UpgradeCatalog251.class); + /** * Constructor. * @@ -76,6 +89,7 @@ public class UpgradeCatalog251 extends AbstractUpgradeCatalog { @Override protected void executeDDLUpdates() throws AmbariException, SQLException { addBackgroundColumnToHostRoleCommand(); + moveClusterHostColumnFromStageToRequest(); } /** @@ -136,4 +150,19 @@ public class UpgradeCatalog251 extends AbstractUpgradeCatalog { dbAccessor.addColumn(HOST_ROLE_COMMAND_TABLE, new DBColumnInfo(HRC_IS_BACKGROUND_COLUMN, Short.class, null, 0, false)); } + + /** + * Moves the {@value #CLUSTER_HOST_INFO_COLUMN} column from {@value #STAGE_TABLE} table to the + * {@value #REQUEST_TABLE} table + * + * + * @throws SQLException + */ + private void moveClusterHostColumnFromStageToRequest() throws SQLException { + DBColumnInfo sourceColumn = new DBColumnInfo(CLUSTER_HOST_INFO_COLUMN, byte[].class, null, null, false); + DBColumnInfo targetColumn = new DBColumnInfo(CLUSTER_HOST_INFO_COLUMN, byte[].class, null, null, false); + + dbAccessor.moveColumnToAnotherTable(STAGE_TABLE, sourceColumn, REQUEST_ID_COLUMN, REQUEST_TABLE, targetColumn, + REQUEST_ID_COLUMN, false); + } } http://git-wip-us.apache.org/repos/asf/ambari/blob/f2bbe478/ambari-server/src/main/java/org/apache/ambari/server/utils/StageUtils.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/utils/StageUtils.java b/ambari-server/src/main/java/org/apache/ambari/server/utils/StageUtils.java index 6a88aea..f184f37 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/utils/StageUtils.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/utils/StageUtils.java @@ -194,20 +194,20 @@ public class StageUtils { return requestStageIds; } - public static Stage getATestStage(long requestId, long stageId, String clusterHostInfo, String commandParamsStage, String hostParamsStage) { + public static Stage getATestStage(long requestId, long stageId, String commandParamsStage, String hostParamsStage) { String hostname; try { hostname = InetAddress.getLocalHost().getHostName(); } catch (UnknownHostException e) { hostname = "host-dummy"; } - return getATestStage(requestId, stageId, hostname, clusterHostInfo, commandParamsStage, hostParamsStage); + return getATestStage(requestId, stageId, hostname, commandParamsStage, hostParamsStage); } //For testing only @Inject - public static Stage getATestStage(long requestId, long stageId, String hostname, String clusterHostInfo, String commandParamsStage, String hostParamsStage) { - Stage s = stageFactory.createNew(requestId, "/tmp", "cluster1", 1L, "context", clusterHostInfo, commandParamsStage, hostParamsStage); + public static Stage getATestStage(long requestId, long stageId, String hostname, String commandParamsStage, String hostParamsStage) { + Stage s = stageFactory.createNew(requestId, "/tmp", "cluster1", 1L, "context", commandParamsStage, hostParamsStage); s.setStageId(stageId); long now = System.currentTimeMillis(); s.addHostRoleExecutionCommand(hostname, Role.NAMENODE, RoleCommand.INSTALL, http://git-wip-us.apache.org/repos/asf/ambari/blob/f2bbe478/ambari-server/src/main/resources/Ambari-DDL-Derby-CREATE.sql ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/Ambari-DDL-Derby-CREATE.sql b/ambari-server/src/main/resources/Ambari-DDL-Derby-CREATE.sql index ece6600..15670f3 100644 --- a/ambari-server/src/main/resources/Ambari-DDL-Derby-CREATE.sql +++ b/ambari-server/src/main/resources/Ambari-DDL-Derby-CREATE.sql @@ -341,6 +341,7 @@ CREATE TABLE request ( start_time BIGINT NOT NULL, status VARCHAR(255) NOT NULL DEFAULT 'PENDING', display_status VARCHAR(255) NOT NULL DEFAULT 'PENDING', + cluster_host_info BLOB NOT NULL, CONSTRAINT PK_request PRIMARY KEY (request_id), CONSTRAINT FK_request_schedule_id FOREIGN KEY (request_schedule_id) REFERENCES requestschedule (schedule_id)); @@ -352,7 +353,6 @@ CREATE TABLE stage ( supports_auto_skip_failure SMALLINT DEFAULT 0 NOT NULL, log_info VARCHAR(255) NOT NULL, request_context VARCHAR(255), - cluster_host_info BLOB NOT NULL, command_params BLOB, host_params BLOB, command_execution_type VARCHAR(32) NOT NULL DEFAULT 'STAGE', http://git-wip-us.apache.org/repos/asf/ambari/blob/f2bbe478/ambari-server/src/main/resources/Ambari-DDL-MySQL-CREATE.sql ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/Ambari-DDL-MySQL-CREATE.sql b/ambari-server/src/main/resources/Ambari-DDL-MySQL-CREATE.sql index e0f2ef9..7e41399 100644 --- a/ambari-server/src/main/resources/Ambari-DDL-MySQL-CREATE.sql +++ b/ambari-server/src/main/resources/Ambari-DDL-MySQL-CREATE.sql @@ -362,6 +362,7 @@ CREATE TABLE request ( start_time BIGINT NOT NULL, status VARCHAR(255) NOT NULL DEFAULT 'PENDING', display_status VARCHAR(255) NOT NULL DEFAULT 'PENDING', + cluster_host_info LONGBLOB, CONSTRAINT PK_request PRIMARY KEY (request_id), CONSTRAINT FK_request_schedule_id FOREIGN KEY (request_schedule_id) REFERENCES requestschedule (schedule_id)); @@ -373,7 +374,6 @@ CREATE TABLE stage ( supports_auto_skip_failure SMALLINT DEFAULT 0 NOT NULL, log_info VARCHAR(255) NOT NULL, request_context VARCHAR(255), - cluster_host_info LONGBLOB, command_params LONGBLOB, host_params LONGBLOB, command_execution_type VARCHAR(32) NOT NULL DEFAULT 'STAGE', http://git-wip-us.apache.org/repos/asf/ambari/blob/f2bbe478/ambari-server/src/main/resources/Ambari-DDL-Oracle-CREATE.sql ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/Ambari-DDL-Oracle-CREATE.sql b/ambari-server/src/main/resources/Ambari-DDL-Oracle-CREATE.sql index 6d0f856..4d0274f 100644 --- a/ambari-server/src/main/resources/Ambari-DDL-Oracle-CREATE.sql +++ b/ambari-server/src/main/resources/Ambari-DDL-Oracle-CREATE.sql @@ -342,6 +342,7 @@ CREATE TABLE request ( start_time NUMBER(19) NOT NULL, status VARCHAR(255) NOT NULL DEFAULT 'PENDING', display_status VARCHAR(255) NOT NULL DEFAULT 'PENDING', + cluster_host_info BLOB NOT NULL, CONSTRAINT PK_request PRIMARY KEY (request_id), CONSTRAINT FK_request_schedule_id FOREIGN KEY (request_schedule_id) REFERENCES requestschedule (schedule_id)); @@ -353,7 +354,6 @@ CREATE TABLE stage ( supports_auto_skip_failure NUMBER(1) DEFAULT 0 NOT NULL, log_info VARCHAR2(255) NULL, request_context VARCHAR2(255) NULL, - cluster_host_info BLOB NOT NULL, command_params BLOB, host_params BLOB, command_execution_type VARCHAR2(32) DEFAULT 'STAGE' NOT NULL, http://git-wip-us.apache.org/repos/asf/ambari/blob/f2bbe478/ambari-server/src/main/resources/Ambari-DDL-Postgres-CREATE.sql ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/Ambari-DDL-Postgres-CREATE.sql b/ambari-server/src/main/resources/Ambari-DDL-Postgres-CREATE.sql index 49b956b..cc933fa 100644 --- a/ambari-server/src/main/resources/Ambari-DDL-Postgres-CREATE.sql +++ b/ambari-server/src/main/resources/Ambari-DDL-Postgres-CREATE.sql @@ -341,6 +341,7 @@ CREATE TABLE request ( start_time BIGINT NOT NULL, status VARCHAR(255) NOT NULL DEFAULT 'PENDING', display_status VARCHAR(255) NOT NULL DEFAULT 'PENDING', + cluster_host_info BYTEA NOT NULL, CONSTRAINT PK_request PRIMARY KEY (request_id), CONSTRAINT FK_request_schedule_id FOREIGN KEY (request_schedule_id) REFERENCES requestschedule (schedule_id)); @@ -352,7 +353,6 @@ CREATE TABLE stage ( supports_auto_skip_failure SMALLINT DEFAULT 0 NOT NULL, log_info VARCHAR(255) NOT NULL, request_context VARCHAR(255), - cluster_host_info BYTEA NOT NULL, command_params BYTEA, host_params BYTEA, command_execution_type VARCHAR(32) DEFAULT 'STAGE' NOT NULL, http://git-wip-us.apache.org/repos/asf/ambari/blob/f2bbe478/ambari-server/src/main/resources/Ambari-DDL-SQLAnywhere-CREATE.sql ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/Ambari-DDL-SQLAnywhere-CREATE.sql b/ambari-server/src/main/resources/Ambari-DDL-SQLAnywhere-CREATE.sql index c594a2e..5fc14d4 100644 --- a/ambari-server/src/main/resources/Ambari-DDL-SQLAnywhere-CREATE.sql +++ b/ambari-server/src/main/resources/Ambari-DDL-SQLAnywhere-CREATE.sql @@ -340,6 +340,7 @@ CREATE TABLE request ( start_time NUMERIC(19) NOT NULL, status VARCHAR(255) NOT NULL DEFAULT 'PENDING', display_status VARCHAR(255) NOT NULL DEFAULT 'PENDING', + cluster_host_info IMAGE, CONSTRAINT PK_request PRIMARY KEY (request_id), CONSTRAINT FK_request_schedule_id FOREIGN KEY (request_schedule_id) REFERENCES requestschedule (schedule_id)); @@ -351,7 +352,6 @@ CREATE TABLE stage ( supports_auto_skip_failure SMALLINT DEFAULT 0 NOT NULL, log_info VARCHAR(255) NOT NULL, request_context VARCHAR(255), - cluster_host_info IMAGE, command_params IMAGE, host_params IMAGE, command_execution_type VARCHAR(32) NOT NULL DEFAULT 'STAGE', http://git-wip-us.apache.org/repos/asf/ambari/blob/f2bbe478/ambari-server/src/main/resources/Ambari-DDL-SQLServer-CREATE.sql ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/Ambari-DDL-SQLServer-CREATE.sql b/ambari-server/src/main/resources/Ambari-DDL-SQLServer-CREATE.sql index 77459a6..12e66f9 100644 --- a/ambari-server/src/main/resources/Ambari-DDL-SQLServer-CREATE.sql +++ b/ambari-server/src/main/resources/Ambari-DDL-SQLServer-CREATE.sql @@ -346,6 +346,7 @@ CREATE TABLE request ( start_time BIGINT NOT NULL, status VARCHAR(255) NOT NULL DEFAULT 'PENDING', display_status VARCHAR(255) NOT NULL DEFAULT 'PENDING', + cluster_host_info VARBINARY(MAX) NOT NULL, CONSTRAINT PK_request PRIMARY KEY CLUSTERED (request_id), CONSTRAINT FK_request_schedule_id FOREIGN KEY (request_schedule_id) REFERENCES requestschedule (schedule_id)); @@ -357,7 +358,6 @@ CREATE TABLE stage ( supports_auto_skip_failure SMALLINT DEFAULT 0 NOT NULL, log_info VARCHAR(255) NOT NULL, request_context VARCHAR(255), - cluster_host_info VARBINARY(MAX) NOT NULL, command_params VARBINARY(MAX), host_params VARBINARY(MAX), command_execution_type VARCHAR(32) NOT NULL DEFAULT 'STAGE', http://git-wip-us.apache.org/repos/asf/ambari/blob/f2bbe478/ambari-server/src/main/resources/properties.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/properties.json b/ambari-server/src/main/resources/properties.json index e536d05..e0f81e6 100644 --- a/ambari-server/src/main/resources/properties.json +++ b/ambari-server/src/main/resources/properties.json @@ -139,6 +139,7 @@ "Requests/abort_reason", "Requests/remove_pending_host_requests", "Requests/pending_host_request_count", + "Requests/cluster_host_info", "_" ], "RequestSchedule" : [ http://git-wip-us.apache.org/repos/asf/ambari/blob/f2bbe478/ambari-server/src/test/java/org/apache/ambari/server/actionmanager/ExecutionCommandWrapperTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/actionmanager/ExecutionCommandWrapperTest.java b/ambari-server/src/test/java/org/apache/ambari/server/actionmanager/ExecutionCommandWrapperTest.java index 89ec32b..8165da5 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/actionmanager/ExecutionCommandWrapperTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/actionmanager/ExecutionCommandWrapperTest.java @@ -149,7 +149,7 @@ public class ExecutionCommandWrapperTest { private static void createTask(ActionDBAccessor db, long requestId, long stageId, String hostName, String clusterName) throws AmbariException { - Stage s = stageFactory.createNew(requestId, "/var/log", clusterName, 1L, "execution command wrapper test", "clusterHostInfo", "commandParamsStage", "hostParamsStage"); + Stage s = stageFactory.createNew(requestId, "/var/log", clusterName, 1L, "execution command wrapper test", "commandParamsStage", "hostParamsStage"); s.setStageId(stageId); s.addHostRoleExecutionCommand(hostName, Role.NAMENODE, RoleCommand.START, @@ -157,7 +157,7 @@ public class ExecutionCommandWrapperTest { hostName, System.currentTimeMillis()), clusterName, "HDFS", false, false); List<Stage> stages = new ArrayList<>(); stages.add(s); - Request request = new Request(stages, clusters); + Request request = new Request(stages, "clusterHostInfo", clusters); db.persistActions(request); } http://git-wip-us.apache.org/repos/asf/ambari/blob/f2bbe478/ambari-server/src/test/java/org/apache/ambari/server/actionmanager/StageTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/actionmanager/StageTest.java b/ambari-server/src/test/java/org/apache/ambari/server/actionmanager/StageTest.java index 89627f7..214aee1 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/actionmanager/StageTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/actionmanager/StageTest.java @@ -66,7 +66,7 @@ public class StageTest { @Test public void testAddServerActionCommand_userName() throws Exception { - final Stage stage = stageFactory.createNew(1, "/tmp", "cluster1", 978, "context", CLUSTER_HOST_INFO, + final Stage stage = stageFactory.createNew(1, "/tmp", "cluster1", 978, "context", "{\"host_param\":\"param_value\"}", "{\"stage_param\":\"param_value\"}"); stage.addServerActionCommand(ConfigureAction.class.getName(),