AMBARI-13032 - Automatically Skip Failed Tasks Of Slaves During Upgrade (jonathanhurley)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/8def5a40 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/8def5a40 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/8def5a40 Branch: refs/heads/trunk Commit: 8def5a407399c56c51ad4edca1a59377dc3c3ba6 Parents: 577ae0c Author: Jonathan Hurley <[email protected]> Authored: Tue Sep 8 09:27:20 2015 -0400 Committer: Jonathan Hurley <[email protected]> Committed: Tue Sep 8 17:15:22 2015 -0400 ---------------------------------------------------------------------- .../actionmanager/ActionDBAccessorImpl.java | 44 +++++-- .../server/actionmanager/HostRoleCommand.java | 18 ++- .../actionmanager/HostRoleCommandFactory.java | 65 +++++++--- .../HostRoleCommandFactoryImpl.java | 50 +++----- .../server/actionmanager/HostRoleStatus.java | 27 +++- .../ambari/server/actionmanager/Stage.java | 123 ++++++++++--------- .../controller/ActionExecutionContext.java | 52 +++++++- .../controller/AmbariActionExecutionHelper.java | 13 +- .../AmbariCustomCommandExecutionHelper.java | 83 +++++-------- .../AmbariManagementControllerImpl.java | 49 ++++---- .../server/controller/KerberosHelperImpl.java | 19 ++- .../ClusterStackVersionResourceProvider.java | 2 +- .../HostStackVersionResourceProvider.java | 2 +- .../internal/UpgradeResourceProvider.java | 58 +++++++-- .../server/orm/dao/HostRoleCommandDAO.java | 4 +- .../dao/HostRoleCommandStatusSummaryDTO.java | 17 ++- .../orm/entities/HostRoleCommandEntity.java | 27 ++++ .../ambari/server/state/UpgradeContext.java | 62 +++++++++- .../ambari/server/topology/LogicalRequest.java | 31 +++-- .../server/upgrade/UpgradeCatalog212.java | 38 +++--- .../apache/ambari/server/utils/StageUtils.java | 4 +- .../main/resources/Ambari-DDL-MySQL-CREATE.sql | 1 + .../main/resources/Ambari-DDL-Oracle-CREATE.sql | 1 + .../resources/Ambari-DDL-Postgres-CREATE.sql | 1 + .../Ambari-DDL-Postgres-EMBEDDED-CREATE.sql | 1 + .../resources/Ambari-DDL-SQLAnywhere-CREATE.sql | 1 + .../resources/Ambari-DDL-SQLServer-CREATE.sql | 1 + .../ExecutionCommandWrapperTest.java | 2 +- .../ambari/server/actionmanager/StageTest.java | 4 +- .../actionmanager/TestActionDBAccessorImpl.java | 18 +-- .../server/actionmanager/TestActionManager.java | 2 +- .../actionmanager/TestActionScheduler.java | 16 +-- .../ambari/server/actionmanager/TestStage.java | 4 +- .../server/agent/TestHeartbeatHandler.java | 12 +- .../AmbariManagementControllerTest.java | 21 ++-- .../internal/CalculatedStatusTest.java | 17 ++- .../internal/StageResourceProviderTest.java | 10 +- .../serveraction/ServerActionExecutorTest.java | 4 +- .../server/stageplanner/TestStagePlanner.java | 32 ++--- .../server/upgrade/UpgradeCatalog212Test.java | 108 +++++++++++----- 40 files changed, 680 insertions(+), 364 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/8def5a40/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ActionDBAccessorImpl.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ActionDBAccessorImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ActionDBAccessorImpl.java index 62f8be9..3cd0681 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ActionDBAccessorImpl.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ActionDBAccessorImpl.java @@ -392,18 +392,29 @@ public class ActionDBAccessorImpl implements ActionDBAccessor { List<HostRoleCommandEntity> commandEntities = hostRoleCommandDAO.findByPKs(taskReports.keySet()); for (HostRoleCommandEntity commandEntity : commandEntities) { CommandReport report = taskReports.get(commandEntity.getTaskId()); - if (commandEntity.getStatus() != HostRoleStatus.ABORTED) { - // We don't want to overwrite statuses for ABORTED tasks with - // statuses that have been received from the agent after aborting task - HostRoleStatus status = HostRoleStatus.valueOf(report.getStatus()); - // if FAILED and marked for holding then set status = HOLDING_FAILED - if (status == HostRoleStatus.FAILED && commandEntity.isRetryAllowed()) { - status = HostRoleStatus.HOLDING_FAILED; - } - commandEntity.setStatus(status); - } else { - abortedCommandUpdates.add(commandEntity.getTaskId()); + + switch (commandEntity.getStatus()) { + case ABORTED: + // We don't want to overwrite statuses for ABORTED tasks with + // statuses that have been received from the agent after aborting task + abortedCommandUpdates.add(commandEntity.getTaskId()); + break; + default: + HostRoleStatus status = HostRoleStatus.valueOf(report.getStatus()); + // if FAILED and marked for holding then set status = HOLDING_FAILED + if (status == HostRoleStatus.FAILED && commandEntity.isRetryAllowed()) { + status = HostRoleStatus.HOLDING_FAILED; + + // tasks can be marked as skipped when they fail + if (commandEntity.isFailureAutoSkipped()) { + status = HostRoleStatus.SKIPPED_FAILED; + } + } + + commandEntity.setStatus(status); + break; } + commandEntity.setStdOut(report.getStdOut().getBytes()); commandEntity.setStdError(report.getStdErr().getBytes()); commandEntity.setStructuredOut(report.getStructuredOut() == null ? null : @@ -441,20 +452,30 @@ public class ActionDBAccessorImpl implements ActionDBAccessor { + "HostName " + hostname + " requestId " + requestId + " stageId " + stageId + " role " + role + " report " + report); } + long now = System.currentTimeMillis(); List<HostRoleCommandEntity> commands = hostRoleCommandDAO.findByHostRole( hostname, requestId, stageId, role); + for (HostRoleCommandEntity command : commands) { HostRoleStatus status = HostRoleStatus.valueOf(report.getStatus()); + // if FAILED and marked for holding then set status = HOLDING_FAILED if (status == HostRoleStatus.FAILED && command.isRetryAllowed()) { status = HostRoleStatus.HOLDING_FAILED; + + // tasks can be marked as skipped when they fail + if (command.isFailureAutoSkipped()) { + status = HostRoleStatus.SKIPPED_FAILED; + } } + command.setStatus(status); command.setStdOut(report.getStdOut().getBytes()); command.setStdError(report.getStdErr().getBytes()); command.setStructuredOut(report.getStructuredOut() == null ? null : report.getStructuredOut().getBytes()); + if (HostRoleStatus.getCompletedStates().contains(command.getStatus())) { command.setEndTime(now); if (requestDAO.getLastStageId(requestId).equals(stageId)) { @@ -463,6 +484,7 @@ public class ActionDBAccessorImpl implements ActionDBAccessor { } command.setExitcode(report.getExitCode()); } + hostRoleCommandDAO.mergeAll(commands); if (checkRequest) { http://git-wip-us.apache.org/repos/asf/ambari/blob/8def5a40/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/HostRoleCommand.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/HostRoleCommand.java b/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/HostRoleCommand.java index 9d44454..cd2e528 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/HostRoleCommand.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/HostRoleCommand.java @@ -62,6 +62,7 @@ public class HostRoleCommand { private long lastAttemptTime = -1; private short attemptCount = 0; private final boolean retryAllowed; + private final boolean autoSkipFailure; private RoleCommand roleCommand; private String commandDetail; private String customCommandName; @@ -84,7 +85,7 @@ public class HostRoleCommand { @AssistedInject public HostRoleCommand(String hostName, Role role, ServiceComponentHostEvent event, RoleCommand command, HostDAO hostDAO, ExecutionCommandDAO executionCommandDAO) { - this(hostName, role, event, command, false, hostDAO, executionCommandDAO); + this(hostName, role, event, command, false, false, hostDAO, executionCommandDAO); } /** @@ -97,8 +98,9 @@ public class HostRoleCommand { * @param hostDAO {@link org.apache.ambari.server.orm.dao.HostDAO} instance being injected */ @AssistedInject - public HostRoleCommand(String hostName, Role role, - ServiceComponentHostEvent event, RoleCommand roleCommand, boolean retryAllowed, HostDAO hostDAO, ExecutionCommandDAO executionCommandDAO) { + public HostRoleCommand(String hostName, Role role, ServiceComponentHostEvent event, + RoleCommand roleCommand, boolean retryAllowed, boolean autoSkipFailure, HostDAO hostDAO, + ExecutionCommandDAO executionCommandDAO) { this.hostDAO = hostDAO; this.executionCommandDAO = executionCommandDAO; @@ -106,6 +108,7 @@ public class HostRoleCommand { this.event = new ServiceComponentHostEventWrapper(event); this.roleCommand = roleCommand; this.retryAllowed = retryAllowed; + this.autoSkipFailure = autoSkipFailure; this.hostName = hostName; HostEntity hostEntity = this.hostDAO.findByName(hostName); @@ -116,8 +119,8 @@ public class HostRoleCommand { @AssistedInject public HostRoleCommand(Host host, Role role, ServiceComponentHostEvent event, - RoleCommand roleCommand, - boolean retryAllowed, HostDAO hostDAO, ExecutionCommandDAO executionCommandDAO) { + RoleCommand roleCommand, boolean retryAllowed, boolean autoSkipFailure, HostDAO hostDAO, + ExecutionCommandDAO executionCommandDAO) { this.hostDAO = hostDAO; this.executionCommandDAO = executionCommandDAO; @@ -125,6 +128,7 @@ public class HostRoleCommand { this.event = new ServiceComponentHostEventWrapper(event); this.roleCommand = roleCommand; this.retryAllowed = retryAllowed; + this.autoSkipFailure = autoSkipFailure; hostId = host.getHostId(); hostName = host.getHostName(); } @@ -156,6 +160,7 @@ public class HostRoleCommand { lastAttemptTime = hostRoleCommandEntity.getLastAttemptTime(); attemptCount = hostRoleCommandEntity.getAttemptCount(); retryAllowed = hostRoleCommandEntity.isRetryAllowed(); + autoSkipFailure = hostRoleCommandEntity.isFailureAutoSkipped(); roleCommand = hostRoleCommandEntity.getRoleCommand(); event = new ServiceComponentHostEventWrapper(hostRoleCommandEntity.getEvent()); commandDetail = hostRoleCommandEntity.getCommandDetail(); @@ -177,6 +182,7 @@ public class HostRoleCommand { hostRoleCommandEntity.setLastAttemptTime(lastAttemptTime); hostRoleCommandEntity.setAttemptCount(attemptCount); hostRoleCommandEntity.setRetryAllowed(retryAllowed); + hostRoleCommandEntity.setAutoSkipOnFailure(autoSkipFailure); hostRoleCommandEntity.setRoleCommand(roleCommand); hostRoleCommandEntity.setCommandDetail(commandDetail); hostRoleCommandEntity.setCustomCommandName(customCommandName); @@ -411,6 +417,8 @@ public class HostRoleCommand { builder.append(" Role: ").append(role).append("\n"); builder.append(" Status: ").append(status).append("\n"); builder.append(" Event: ").append(event).append("\n"); + builder.append(" RetryAllowed: ").append(retryAllowed).append("\n"); + builder.append(" AutoSkipFailure: ").append(autoSkipFailure).append("\n"); builder.append(" Output log: ").append(outputLog).append("\n"); builder.append(" Error log: ").append(errorLog).append("\n"); builder.append(" stdout: ").append(stdout).append("\n"); http://git-wip-us.apache.org/repos/asf/ambari/blob/8def5a40/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/HostRoleCommandFactory.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/HostRoleCommandFactory.java b/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/HostRoleCommandFactory.java index 84c2d2a..33da3f0 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/HostRoleCommandFactory.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/HostRoleCommandFactory.java @@ -28,40 +28,67 @@ public interface HostRoleCommandFactory { /** * Constructor via factory. - * @param hostName Host name - * @param role Action to run - * @param event Event on the host and component - * @param command Type of command + * + * @param hostName + * Host name + * @param role + * Action to run + * @param event + * Event on the host and component + * @param command + * Type of command * @return An instance constructed where retryAllowed defaults to false */ - HostRoleCommand create(String hostName, Role role, ServiceComponentHostEvent event, RoleCommand command); + HostRoleCommand create(String hostName, Role role, ServiceComponentHostEvent event, + RoleCommand command); /** * Constructor via factory. - * @param hostName Host name - * @param role Action to run - * @param event Event on the host and component - * @param command Type of command - * @param retryAllowed Whether the command can be repeated + * + * @param hostName + * Host name + * @param role + * Action to run + * @param event + * Event on the host and component + * @param command + * Type of command + * @param retryAllowed + * Whether the command can be repeated + * @param autoSkipFailure + * {@code true} if the command should be automatically skipped if it + * fails. * @return An instance of a HostRoleCommand. */ - HostRoleCommand create(String hostName, Role role, ServiceComponentHostEvent event, RoleCommand command, boolean retryAllowed); + HostRoleCommand create(String hostName, Role role, ServiceComponentHostEvent event, + RoleCommand command, boolean retryAllowed, boolean autoSkipFailure); /** * Constructor via factory. - * @param host Host object - * @param role Action to run - * @param event Event on the host and component - * @param command Type of command - * @param retryAllowed Whether the command can be repeated + * + * @param host + * Host object + * @param role + * Action to run + * @param event + * Event on the host and component + * @param command + * Type of command + * @param retryAllowed + * Whether the command can be repeated + * @param autoSkipFailure + * {@code true} if the command should be automatically skipped if it + * fails. * @return An instance of a HostRoleCommand. */ - HostRoleCommand create(Host host, Role role, ServiceComponentHostEvent event, RoleCommand command, boolean - retryAllowed); + HostRoleCommand create(Host host, Role role, ServiceComponentHostEvent event, RoleCommand command, + boolean retryAllowed, boolean autoSkipFailure); /** * Constructor via factory - * @param hostRoleCommandEntity Object to copy fields from. + * + * @param hostRoleCommandEntity + * Object to copy fields from. * @return An instance constructed from the input object. */ HostRoleCommand createExisting(HostRoleCommandEntity hostRoleCommandEntity); http://git-wip-us.apache.org/repos/asf/ambari/blob/8def5a40/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/HostRoleCommandFactoryImpl.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/HostRoleCommandFactoryImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/HostRoleCommandFactoryImpl.java index 0440f87..cf1e989 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/HostRoleCommandFactoryImpl.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/HostRoleCommandFactoryImpl.java @@ -18,9 +18,6 @@ package org.apache.ambari.server.actionmanager; -import com.google.inject.Inject; -import com.google.inject.Injector; -import com.google.inject.Singleton; import org.apache.ambari.server.Role; import org.apache.ambari.server.RoleCommand; import org.apache.ambari.server.orm.dao.ExecutionCommandDAO; @@ -29,9 +26,13 @@ import org.apache.ambari.server.orm.entities.HostRoleCommandEntity; import org.apache.ambari.server.state.Host; import org.apache.ambari.server.state.ServiceComponentHostEvent; +import com.google.inject.Inject; +import com.google.inject.Injector; +import com.google.inject.Singleton; + @Singleton public class HostRoleCommandFactoryImpl implements HostRoleCommandFactory { - + private Injector injector; @Inject @@ -51,41 +52,28 @@ public class HostRoleCommandFactoryImpl implements HostRoleCommandFactory { public HostRoleCommand create(String hostName, Role role, ServiceComponentHostEvent event, RoleCommand command) { return new HostRoleCommand(hostName, role, event, command, - this.injector.getInstance(HostDAO.class), - this.injector.getInstance(ExecutionCommandDAO.class)); + injector.getInstance(HostDAO.class), + injector.getInstance(ExecutionCommandDAO.class)); } /** - * Constructor via factory. - * @param hostName Host name - * @param role Action to run - * @param event Event on the host and component - * @param command Type of command - * @param retryAllowed Whether the command can be repeated - * @return An instance of a HostRoleCommand. + * {@inheritDoc} */ @Override - public HostRoleCommand create(String hostName, Role role, - ServiceComponentHostEvent event, RoleCommand command, boolean retryAllowed) { - return new HostRoleCommand(hostName, role, event, command, retryAllowed, - this.injector.getInstance(HostDAO.class), - this.injector.getInstance(ExecutionCommandDAO.class)); + public HostRoleCommand create(String hostName, Role role, ServiceComponentHostEvent event, + RoleCommand command, boolean retryAllowed, boolean autoSkipFailure) { + return new HostRoleCommand(hostName, role, event, command, retryAllowed, autoSkipFailure, + injector.getInstance(HostDAO.class), injector.getInstance(ExecutionCommandDAO.class)); } /** - * Constructor via factory. - * @param host Host object - * @param role Action to run - * @param event Event on the host and component - * @param command Type of command - * @param retryAllowed Whether the command can be repeated - * @return An instance of a HostRoleCommand. + * {@inheritDoc} */ @Override - public HostRoleCommand create(Host host, Role role, ServiceComponentHostEvent event, RoleCommand command, boolean retryAllowed) { - return new HostRoleCommand(host, role, event, command, retryAllowed, - this.injector.getInstance(HostDAO.class), - this.injector.getInstance(ExecutionCommandDAO.class)); + public HostRoleCommand create(Host host, Role role, ServiceComponentHostEvent event, + RoleCommand command, boolean retryAllowed, boolean autoSkipFailure) { + return new HostRoleCommand(host, role, event, command, retryAllowed, autoSkipFailure, + injector.getInstance(HostDAO.class), injector.getInstance(ExecutionCommandDAO.class)); } /** @@ -96,7 +84,7 @@ public class HostRoleCommandFactoryImpl implements HostRoleCommandFactory { @Override public HostRoleCommand createExisting(HostRoleCommandEntity hostRoleCommandEntity) { return new HostRoleCommand(hostRoleCommandEntity, - this.injector.getInstance(HostDAO.class), - this.injector.getInstance(ExecutionCommandDAO.class)); + injector.getInstance(HostDAO.class), + injector.getInstance(ExecutionCommandDAO.class)); } } http://git-wip-us.apache.org/repos/asf/ambari/blob/8def5a40/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/HostRoleStatus.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/HostRoleStatus.java b/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/HostRoleStatus.java index 39cbabc..52523c7 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/HostRoleStatus.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/HostRoleStatus.java @@ -27,52 +27,69 @@ public enum HostRoleStatus { * Not queued for a host. */ PENDING, + /** * Queued for a host, or has already been sent to host, but host did not answer yet. */ QUEUED, + /** * Host reported it is working, received an IN_PROGRESS command status from host. */ IN_PROGRESS, + /** * Task is holding, waiting for command to proceed to completion. */ HOLDING, + /** * Host reported success */ COMPLETED, + /** * Failed */ FAILED, + /** * Task is holding after a failure, waiting for command to skip or retry. */ HOLDING_FAILED, + /** * Host did not respond in time */ TIMEDOUT, + /** * Task is holding after a time-out, waiting for command to skip or retry. */ HOLDING_TIMEDOUT, + /** * Operation was abandoned */ - ABORTED; + ABORTED, + + /** + * The operation failed and was automatically skipped. + */ + SKIPPED_FAILED; + + private static List<HostRoleStatus> COMPLETED_STATES = Arrays.asList(FAILED, TIMEDOUT, ABORTED, + COMPLETED, SKIPPED_FAILED); - private static List<HostRoleStatus> COMPLETED_STATES = Arrays.asList(FAILED, TIMEDOUT, ABORTED, COMPLETED); - private static List<HostRoleStatus> HOLDING_STATES = Arrays.asList(HOLDING, HOLDING_FAILED, HOLDING_TIMEDOUT); + private static List<HostRoleStatus> HOLDING_STATES = Arrays.asList(HOLDING, HOLDING_FAILED, + HOLDING_TIMEDOUT); /** * The {@link HostRoleStatus}s that represent any commands which are * considered to be "Failed". */ - public static EnumSet<HostRoleStatus> FAILED_STATUSES = EnumSet.of(FAILED, - TIMEDOUT, ABORTED); + public static EnumSet<HostRoleStatus> FAILED_STATUSES = EnumSet.of(FAILED, TIMEDOUT, ABORTED, + SKIPPED_FAILED); /** * The {@link HostRoleStatus}s that represent any commands which are http://git-wip-us.apache.org/repos/asf/ambari/blob/8def5a40/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/Stage.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/Stage.java b/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/Stage.java index fcd0324..8b2703c 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/Stage.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/Stage.java @@ -270,21 +270,25 @@ public class Stage { return StageUtils.getActionId(requestId, getStageId()); } - private synchronized ExecutionCommandWrapper addGenericExecutionCommand( - String clusterName, String hostName, Role role, - RoleCommand command, ServiceComponentHostEvent event, boolean retryAllowed){ + private synchronized ExecutionCommandWrapper addGenericExecutionCommand(String clusterName, + String hostName, Role role, RoleCommand command, ServiceComponentHostEvent event, + boolean retryAllowed, boolean autoSkipFailure) { + + // used on stage creation only, no need to check if wrappers loaded + HostRoleCommand hrc = hostRoleCommandFactory.create(hostName, role, event, command, + retryAllowed, autoSkipFailure); - //used on stage creation only, no need to check if wrappers loaded - HostRoleCommand hrc = hostRoleCommandFactory.create(hostName, role, event, command, retryAllowed); return addGenericExecutionCommand(clusterName, hostName, role, command, event, hrc); } - private ExecutionCommandWrapper addGenericExecutionCommand( - Cluster cluster, Host host, Role role, - RoleCommand command, ServiceComponentHostEvent event, boolean retryAllowed) { - HostRoleCommand hrc = hostRoleCommandFactory.create(host, role, event, command, retryAllowed); - return addGenericExecutionCommand(cluster.getClusterName(), host.getHostName(), role, command, event, hrc); + private ExecutionCommandWrapper addGenericExecutionCommand(Cluster cluster, Host host, Role role, + RoleCommand command, ServiceComponentHostEvent event, boolean retryAllowed, + boolean autoSkipFailure) { + HostRoleCommand hrc = hostRoleCommandFactory.create(host, role, event, command, retryAllowed, + autoSkipFailure); + return addGenericExecutionCommand(cluster.getClusterName(), host.getHostName(), role, command, + event, hrc); } //TODO refactor method to use Host object (host_id support) @@ -328,33 +332,31 @@ public class Stage { } /** - * A new host role command is created for execution. - * Creates both ExecutionCommand and HostRoleCommand objects and - * adds them to the Stage. This should be called only once for a host-role - * for a given stage. + * A new host role command is created for execution. Creates both + * ExecutionCommand and HostRoleCommand objects and adds them to the Stage. + * This should be called only once for a host-role for a given stage. */ public synchronized void addHostRoleExecutionCommand(String host, Role role, RoleCommand command, - ServiceComponentHostEvent event, String clusterName, - String serviceName, boolean retryAllowed) { + ServiceComponentHostEvent event, String clusterName, String serviceName, boolean retryAllowed, + boolean autoSkipFailure) { - ExecutionCommandWrapper commandWrapper = - addGenericExecutionCommand(clusterName, host, role, command, event, retryAllowed); + ExecutionCommandWrapper commandWrapper = addGenericExecutionCommand(clusterName, host, role, + command, event, retryAllowed, autoSkipFailure); commandWrapper.getExecutionCommand().setServiceName(serviceName); } /** - * A new host role command is created for execution. - * Creates both ExecutionCommand and HostRoleCommand objects and - * adds them to the Stage. This should be called only once for a host-role - * for a given stage. + * A new host role command is created for execution. Creates both + * ExecutionCommand and HostRoleCommand objects and adds them to the Stage. + * This should be called only once for a host-role for a given stage. */ public synchronized void addHostRoleExecutionCommand(Host host, Role role, RoleCommand command, - ServiceComponentHostEvent event, Cluster cluster, - String serviceName, boolean retryAllowed) { + ServiceComponentHostEvent event, Cluster cluster, String serviceName, boolean retryAllowed, + boolean autoSkipFailure) { - ExecutionCommandWrapper commandWrapper = - addGenericExecutionCommand(cluster, host, role, command, event, retryAllowed); + ExecutionCommandWrapper commandWrapper = addGenericExecutionCommand(cluster, host, role, + command, event, retryAllowed, autoSkipFailure); commandWrapper.getExecutionCommand().setServiceName(serviceName); } @@ -364,36 +366,47 @@ public class Stage { * Creates server-side execution command. * <p/> * The action name for this command is expected to be the classname of a - * {@link org.apache.ambari.server.serveraction.ServerAction} implementation which will be - * instantiated and invoked as needed. + * {@link org.apache.ambari.server.serveraction.ServerAction} implementation + * which will be instantiated and invoked as needed. * - * @param actionName a String declaring the action name (in the form of a classname) to execute - * @param userName the name of the user who created this stage; may be null for anonymous user - * @param role the Role for this command - * @param command the RoleCommand for this command - * @param clusterName a String identifying the cluster on which to to execute this command - * @param event a ServiceComponentHostServerActionEvent - * @param commandParams a Map of String to String data used to pass to the action - this may be -* empty or null if no data is relevant - * @param commandDetail a String declaring a descriptive name to pass to the action - null or an -* empty string indicates no value is to be set - * @param configTags a Map of configuration tags to set for this command - if null, no -* configurations will be available for the command - * @param timeout an Integer declaring the timeout for this action - if null, a default - * @param retryAllowed indicates whether retry after failure is allowed + * @param actionName + * a String declaring the action name (in the form of a classname) to + * execute + * @param userName + * the name of the user who created this stage; may be null for + * anonymous user + * @param role + * the Role for this command + * @param command + * the RoleCommand for this command + * @param clusterName + * a String identifying the cluster on which to to execute this + * command + * @param event + * a ServiceComponentHostServerActionEvent + * @param commandParams + * a Map of String to String data used to pass to the action - this + * may be empty or null if no data is relevant + * @param commandDetail + * a String declaring a descriptive name to pass to the action - null + * or an empty string indicates no value is to be set + * @param configTags + * a Map of configuration tags to set for this command - if null, no + * configurations will be available for the command + * @param timeout + * an Integer declaring the timeout for this action - if null, a + * default + * @param retryAllowed + * indicates whether retry after failure is allowed */ - public synchronized void addServerActionCommand(String actionName, - @Nullable String userName, - Role role, RoleCommand command, - String clusterName, - ServiceComponentHostServerActionEvent event, - @Nullable Map<String, String> commandParams, - @Nullable String commandDetail, - @Nullable Map<String, Map<String, String>> configTags, - @Nullable Integer timeout, boolean retryAllowed) { - - ExecutionCommandWrapper commandWrapper = - addGenericExecutionCommand(clusterName, INTERNAL_HOSTNAME, role, command, event, retryAllowed); + public synchronized void addServerActionCommand(String actionName, @Nullable String userName, + Role role, RoleCommand command, String clusterName, + ServiceComponentHostServerActionEvent event, @Nullable Map<String, String> commandParams, + @Nullable String commandDetail, @Nullable Map<String, Map<String, String>> configTags, + @Nullable Integer timeout, boolean retryAllowed, boolean autoSkipFailure) { + + ExecutionCommandWrapper commandWrapper = addGenericExecutionCommand(clusterName, + INTERNAL_HOSTNAME, role, command, event, retryAllowed, autoSkipFailure); ExecutionCommand cmd = commandWrapper.getExecutionCommand(); @@ -439,7 +452,7 @@ public class Stage { */ public synchronized void addCancelRequestCommand(List<Long> cancelTargets, String clusterName, String hostName) { ExecutionCommandWrapper commandWrapper = addGenericExecutionCommand(clusterName, hostName, - Role.AMBARI_SERVER_ACTION, RoleCommand.ABORT, null, false); + Role.AMBARI_SERVER_ACTION, RoleCommand.ABORT, null, false, false); ExecutionCommand cmd = commandWrapper.getExecutionCommand(); cmd.setCommandType(AgentCommandType.CANCEL_COMMAND); http://git-wip-us.apache.org/repos/asf/ambari/blob/8def5a40/ambari-server/src/main/java/org/apache/ambari/server/controller/ActionExecutionContext.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/ActionExecutionContext.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/ActionExecutionContext.java index ee5febe..dc88a5f 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/ActionExecutionContext.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/ActionExecutionContext.java @@ -40,6 +40,14 @@ public class ActionExecutionContext { private String expectedServiceName; private String expectedComponentName; private boolean ignoreMaintenance = false; + private boolean allowRetry = false; + + /** + * {@code true} if slave/client component failures should be automatically + * skipped. This will only automatically skip the failure if the task is + * skippable to begin with. + */ + private boolean autoSkipFailures = false; /** * Create an ActionExecutionContext to execute an action from a request @@ -71,7 +79,7 @@ public class ActionExecutionContext { List<RequestResourceFilter> resourceFilters, Map<String, String> parameters) { this.clusterName = clusterName; - this.actionName = commandName; + actionName = commandName; this.resourceFilters = resourceFilters; this.parameters = parameters; } @@ -120,6 +128,46 @@ public class ActionExecutionContext { return expectedComponentName; } + /** + * Gets whether the action can be retried if it failed. The default is + * {@code true)}. + * + * @return {@code true} if the action can be retried if it fails. + */ + public boolean isRetryAllowed() { + return allowRetry; + } + + /** + * Sets whether the action can be retried if it fails. + * + * @param allowRetry + * {@code true} if the action can be retried if it fails. + */ + public void setRetryAllowed(boolean allowRetry){ + this.allowRetry = allowRetry; + } + + /** + * Gets whether skippable actions that failed are automatically skipped. + * + * @return the autoSkipFailures + */ + public boolean isFailureAutoSkipped() { + return autoSkipFailures; + } + + /** + * Sets whether skippable action that failed are automatically skipped. + * + * @param autoSkipFailures + * {@code true} to automatically skip failures which are marked as + * skippable. + */ + public void setAutoSkipFailures(boolean autoSkipFailures) { + this.autoSkipFailures = autoSkipFailures; + } + @Override public String toString() { return "ActionExecutionContext{" + @@ -131,6 +179,8 @@ public class ActionExecutionContext { ", targetType=" + targetType + ", timeout=" + timeout + ", ignoreMaintenance=" + ignoreMaintenance + + ", allowRetry=" + allowRetry + + ", autoSkipFailures=" + autoSkipFailures + '}'; } http://git-wip-us.apache.org/repos/asf/ambari/blob/8def5a40/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariActionExecutionHelper.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariActionExecutionHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariActionExecutionHelper.java index a422b2d..a13e86d 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariActionExecutionHelper.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariActionExecutionHelper.java @@ -215,7 +215,7 @@ public class AmbariActionExecutionHelper { * @throws AmbariException if the task can not be added */ public void addExecutionCommandsToStage( - final ActionExecutionContext actionContext, Stage stage, boolean retryAllowed) +final ActionExecutionContext actionContext, Stage stage) throws AmbariException { String actionName = actionContext.getActionName(); @@ -335,11 +335,12 @@ public class AmbariActionExecutionHelper { // create tasks for each host for (String hostName : targetHosts) { - stage.addHostRoleExecutionCommand(hostName, - Role.valueOf(actionContext.getActionName()), RoleCommand.ACTIONEXECUTE, - new ServiceComponentHostOpInProgressEvent(actionContext.getActionName(), - hostName, System.currentTimeMillis()), clusterName, - serviceName, retryAllowed); + stage.addHostRoleExecutionCommand(hostName, Role.valueOf(actionContext.getActionName()), + RoleCommand.ACTIONEXECUTE, + new ServiceComponentHostOpInProgressEvent(actionContext.getActionName(), hostName, + System.currentTimeMillis()), + clusterName, serviceName, actionContext.isRetryAllowed(), + actionContext.isFailureAutoSkipped()); Map<String, String> commandParams = new TreeMap<String, String>(); int maxTaskTimeout = Integer.parseInt(configs.getDefaultAgentTaskTimeout(false)); http://git-wip-us.apache.org/repos/asf/ambari/blob/8def5a40/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java index 43bdbfe..2bda16e 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java @@ -26,6 +26,7 @@ import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.DB_DRIVER import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.DB_NAME; import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.GROUP_LIST; import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.HOOKS_FOLDER; +import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.HOST_SYS_PREPPED; import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.JAVA_HOME; import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.JAVA_VERSION; import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.JCE_NAME; @@ -33,7 +34,6 @@ import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.JDK_LOCAT import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.JDK_NAME; import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.MYSQL_JDBC_URL; import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.ORACLE_JDBC_URL; -import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.HOST_SYS_PREPPED; import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.REPO_INFO; import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.SCRIPT; import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.SCRIPT_TYPE; @@ -232,15 +232,13 @@ public class AmbariCustomCommandExecutionHelper { } private void addCustomCommandAction(final ActionExecutionContext actionExecutionContext, - final RequestResourceFilter resourceFilter, - Stage stage, - Map<String, String> additionalCommandParams, - String commandDetail, - boolean retryAllowed) - throws AmbariException { + final RequestResourceFilter resourceFilter, Stage stage, + Map<String, String> additionalCommandParams, String commandDetail) throws AmbariException { final String serviceName = resourceFilter.getServiceName(); final String componentName = resourceFilter.getComponentName(); final String commandName = actionExecutionContext.getActionName(); + boolean retryAllowed = actionExecutionContext.isRetryAllowed(); + boolean autoSkipFailure = actionExecutionContext.isFailureAutoSkipped(); String clusterName = stage.getClusterName(); final Cluster cluster = clusters.getCluster(clusterName); @@ -298,8 +296,8 @@ public class AmbariCustomCommandExecutionHelper { stage.addHostRoleExecutionCommand(hostName, Role.valueOf(componentName), RoleCommand.CUSTOM_COMMAND, - new ServiceComponentHostOpInProgressEvent(componentName, - hostName, nowTimestamp), cluster.getClusterName(), serviceName, retryAllowed); + new ServiceComponentHostOpInProgressEvent(componentName, hostName, nowTimestamp), + cluster.getClusterName(), serviceName, retryAllowed, autoSkipFailure); Map<String, Map<String, String>> configurations = new TreeMap<String, Map<String, String>>(); @@ -423,12 +421,8 @@ public class AmbariCustomCommandExecutionHelper { return retVal; } - private void findHostAndAddServiceCheckAction( - final ActionExecutionContext actionExecutionContext, - final RequestResourceFilter resourceFilter, - Stage stage, - boolean retryAllowed) - throws AmbariException { + private void findHostAndAddServiceCheckAction(final ActionExecutionContext actionExecutionContext, + final RequestResourceFilter resourceFilter, Stage stage) throws AmbariException { String clusterName = actionExecutionContext.getClusterName(); final Cluster cluster = clusters.getCluster(clusterName); @@ -500,40 +494,34 @@ public class AmbariCustomCommandExecutionHelper { LOG.info(msg); } - addServiceCheckAction(stage, hostName, smokeTestRole, nowTimestamp, - serviceName, componentName, actionParameters, retryAllowed); + addServiceCheckAction(stage, hostName, smokeTestRole, nowTimestamp, serviceName, componentName, + actionParameters, actionExecutionContext.isRetryAllowed(), + actionExecutionContext.isFailureAutoSkipped()); } /** - * Creates and populates service check EXECUTION_COMMAND for host. - * Not all EXECUTION_COMMAND parameters are populated here because they - * are not needed by service check. + * Creates and populates service check EXECUTION_COMMAND for host. Not all + * EXECUTION_COMMAND parameters are populated here because they are not needed + * by service check. */ - public void addServiceCheckAction(Stage stage, - String hostname, String smokeTestRole, - long nowTimestamp, - String serviceName, - String componentName, - Map<String, String> actionParameters, - boolean retryAllowed) - throws AmbariException { + public void addServiceCheckAction(Stage stage, String hostname, String smokeTestRole, + long nowTimestamp, String serviceName, String componentName, + Map<String, String> actionParameters, boolean retryAllowed, boolean autoSkipFailure) + throws AmbariException { String clusterName = stage.getClusterName(); Cluster cluster = clusters.getCluster(clusterName); StackId stackId = cluster.getDesiredStackVersion(); AmbariMetaInfo ambariMetaInfo = managementController.getAmbariMetaInfo(); - ServiceInfo serviceInfo = - ambariMetaInfo.getService(stackId.getStackName(), - stackId.getStackVersion(), serviceName); + ServiceInfo serviceInfo = ambariMetaInfo.getService(stackId.getStackName(), + stackId.getStackVersion(), serviceName); StackInfo stackInfo = ambariMetaInfo.getStack(stackId.getStackName(), stackId.getStackVersion()); - - stage.addHostRoleExecutionCommand(hostname, - Role.valueOf(smokeTestRole), + stage.addHostRoleExecutionCommand(hostname, Role.valueOf(smokeTestRole), RoleCommand.SERVICE_CHECK, - new ServiceComponentHostOpInProgressEvent(componentName, hostname, - nowTimestamp), cluster.getClusterName(), serviceName, retryAllowed); + new ServiceComponentHostOpInProgressEvent(componentName, hostname, nowTimestamp), + cluster.getClusterName(), serviceName, retryAllowed, autoSkipFailure); HostRoleCommand hrc = stage.getHostRoleCommand(hostname, smokeTestRole); if (hrc != null) { @@ -615,10 +603,7 @@ public class AmbariCustomCommandExecutionHelper { * calls into the implementation of a custom command */ private void addDecommissionAction(final ActionExecutionContext actionExecutionContext, - final RequestResourceFilter resourceFilter, - Stage stage, - boolean retryAllowed) - throws AmbariException { + final RequestResourceFilter resourceFilter, Stage stage) throws AmbariException { String clusterName = actionExecutionContext.getClusterName(); final Cluster cluster = clusters.getCluster(clusterName); @@ -828,7 +813,8 @@ public class AmbariCustomCommandExecutionHelper { if (!serviceName.equals(Service.Type.HBASE.name()) || hostName.equals(primaryCandidate)) { commandParams.put(UPDATE_EXCLUDE_FILE_ONLY, "false"); addCustomCommandAction(commandContext, commandFilter, stage, - commandParams, commandDetail.toString(), retryAllowed); + commandParams, + commandDetail.toString()); } } } @@ -895,10 +881,7 @@ public class AmbariCustomCommandExecutionHelper { * @throws AmbariException if the commands can not be added */ public void addExecutionCommandsToStage(ActionExecutionContext actionExecutionContext, - Stage stage, - Map<String, String> requestParams, - boolean retryAllowed) - throws AmbariException { + Stage stage, Map<String, String> requestParams) throws AmbariException { List<RequestResourceFilter> resourceFilters = actionExecutionContext.getResourceFilters(); @@ -909,12 +892,10 @@ public class AmbariCustomCommandExecutionHelper { + ", request=" + actionExecutionContext.toString()); String actionName = actionExecutionContext.getActionName(); - if (actionName.contains(SERVICE_CHECK_COMMAND_NAME)) { - findHostAndAddServiceCheckAction(actionExecutionContext, - resourceFilter, stage, retryAllowed); + findHostAndAddServiceCheckAction(actionExecutionContext, resourceFilter, stage); } else if (actionName.equals(DECOMMISSION_COMMAND_NAME)) { - addDecommissionAction(actionExecutionContext, resourceFilter, stage, retryAllowed); + addDecommissionAction(actionExecutionContext, resourceFilter, stage); } else if (isValidCustomCommand(actionExecutionContext, resourceFilter)) { String commandDetail = getReadableCustomCommandDetail(actionExecutionContext, resourceFilter); @@ -945,8 +926,8 @@ public class AmbariCustomCommandExecutionHelper { } } - addCustomCommandAction(actionExecutionContext, resourceFilter, stage, - extraParams, commandDetail, retryAllowed); + addCustomCommandAction(actionExecutionContext, resourceFilter, stage, extraParams, + commandDetail); } else { throw new AmbariException("Unsupported action " + actionName); } http://git-wip-us.apache.org/repos/asf/ambari/blob/8def5a40/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java index a90cb31..f27e388 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java @@ -23,10 +23,12 @@ import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.AMBARI_DB import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.AMBARI_DB_RCA_URL; import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.AMBARI_DB_RCA_USERNAME; import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.CLIENTS_TO_UPDATE_CONFIGS; +import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.COMMAND_RETRY_ENABLED; import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.COMMAND_TIMEOUT; import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.DB_DRIVER_FILENAME; import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.GROUP_LIST; import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.HOOKS_FOLDER; +import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.MAX_DURATION_OF_RETRIES; import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.PACKAGE_LIST; import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.REPO_INFO; import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.SCRIPT; @@ -35,8 +37,6 @@ import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.SERVICE_P import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.SERVICE_REPO_INFO; import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.USER_LIST; import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.VERSION; -import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.MAX_DURATION_OF_RETRIES; -import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.COMMAND_RETRY_ENABLED; import java.io.File; import java.io.FileReader; @@ -60,8 +60,6 @@ import java.util.TreeMap; import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.Lock; -import com.google.common.collect.ArrayListMultimap; -import com.google.common.collect.Multimap; import org.apache.ambari.server.AmbariException; import org.apache.ambari.server.ClusterNotFoundException; import org.apache.ambari.server.DuplicateResourceException; @@ -162,23 +160,24 @@ import org.apache.ambari.server.state.svccomphost.ServiceComponentHostStopEvent; import org.apache.ambari.server.state.svccomphost.ServiceComponentHostUpgradeEvent; import org.apache.ambari.server.utils.StageUtils; import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.collections.MultiMap; import org.apache.commons.io.IOUtils; import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.math.NumberUtils; import org.apache.http.client.utils.URIBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.security.core.GrantedAuthority; import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; +import com.google.common.collect.ArrayListMultimap; +import com.google.common.collect.Multimap; import com.google.gson.Gson; import com.google.gson.reflect.TypeToken; import com.google.inject.Inject; import com.google.inject.Injector; import com.google.inject.Singleton; import com.google.inject.persist.Transactional; -import org.springframework.security.core.GrantedAuthority; @Singleton public class AmbariManagementControllerImpl implements AmbariManagementController { @@ -1367,13 +1366,13 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle clusterConfigAttributes = clusterConfig.getPropertiesAttributes(); if (!isAttributeMapsEqual(requestConfigAttributes, clusterConfigAttributes)){ isConfigurationCreationNeeded = true; - break; + break; } } else { isConfigurationCreationNeeded = true; break; } - + if (requestConfigProperties == null || requestConfigProperties.isEmpty()) { Config existingConfig = cluster.getConfig(desiredConfig.getType(), desiredConfig.getVersionTag()); if (existingConfig != null) { @@ -1573,17 +1572,17 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle } /** - * Comparison of two attributes maps + * Comparison of two attributes maps * @param requestConfigAttributes - attribute map sent from API * @param clusterConfigAttributes - existed attribute map - * @return true if maps is equal (have the same attributes and their values) + * @return true if maps is equal (have the same attributes and their values) */ public boolean isAttributeMapsEqual(Map<String, Map<String, String>> requestConfigAttributes, Map<String, Map<String, String>> clusterConfigAttributes) { boolean isAttributesEqual = true; if ((requestConfigAttributes != null && clusterConfigAttributes == null) || (requestConfigAttributes == null && clusterConfigAttributes != null) - || (requestConfigAttributes != null && clusterConfigAttributes != null + || (requestConfigAttributes != null && clusterConfigAttributes != null && !requestConfigAttributes.keySet().equals(clusterConfigAttributes.keySet()))) { return false; } else if (clusterConfigAttributes != null && requestConfigAttributes != null) { @@ -1592,7 +1591,7 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle Map<String, String> requestMapAttributes = requestConfigAttributes.get(ClusterEntrySet.getKey()); if ((requestMapAttributes != null && clusterMapAttributes == null) || (requestMapAttributes == null && clusterMapAttributes != null) - || (requestMapAttributes != null && clusterMapAttributes != null + || (requestMapAttributes != null && clusterMapAttributes != null && !requestMapAttributes.keySet().equals(clusterMapAttributes.keySet()))) { return false; } else if (requestMapAttributes != null && clusterMapAttributes != null) { @@ -1601,7 +1600,7 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle String clusterPropertyValue = clusterMapAttributes.get(requestPropertyEntrySet.getKey()); if ((requestPropertyValue != null && clusterPropertyValue == null) || (requestPropertyValue == null && clusterPropertyValue != null) - || (requestPropertyValue != null && clusterPropertyValue != null + || (requestPropertyValue != null && clusterPropertyValue != null && !requestPropertyValue.equals(clusterPropertyValue))) { return false; } @@ -1611,8 +1610,8 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle } } return isAttributesEqual; - } - + } + /** * Save cluster update results to retrieve later * @param clusterRequest cluster request info @@ -1848,10 +1847,10 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle String serviceName = scHost.getServiceName(); - stage.addHostRoleExecutionCommand(scHost.getHost(), Role.valueOf(scHost - .getServiceComponentName()), roleCommand, - event, cluster, - serviceName, false); + stage.addHostRoleExecutionCommand(scHost.getHost(), + Role.valueOf(scHost.getServiceComponentName()), roleCommand, event, cluster, serviceName, + false, false); + String componentName = scHost.getServiceComponentName(); String hostname = scHost.getHostName(); String osFamily = clusters.getHost(hostname).getOsFamily(); @@ -1885,7 +1884,7 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle LOG.info("Adding service type info in createHostAction:: " + serviceInfo.getServiceType()); execCmd.setServiceType(serviceInfo.getServiceType()); - + execCmd.setConfigurations(configurations); execCmd.setConfigurationAttributes(configurationAttributes); execCmd.setConfigurationTags(configTags); @@ -2439,9 +2438,8 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle continue; } - customCommandExecutionHelper.addServiceCheckAction(stage, clientHost, - smokeTestRole, nowTimestamp, serviceName, - componentName, null, false); + customCommandExecutionHelper.addServiceCheckAction(stage, clientHost, smokeTestRole, + nowTimestamp, serviceName, componentName, null, false, false); } RoleCommandOrder rco = getRoleCommandOrder(cluster); @@ -3376,9 +3374,10 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle jsons.getClusterHostInfo(), commandParamsForStage, jsons.getHostParamsForStage()); if (actionRequest.isCommand()) { - customCommandExecutionHelper.addExecutionCommandsToStage(actionExecContext, stage, requestProperties, false); + customCommandExecutionHelper.addExecutionCommandsToStage(actionExecContext, stage, + requestProperties); } else { - actionExecutionHelper.addExecutionCommandsToStage(actionExecContext, stage, false); + actionExecutionHelper.addExecutionCommandsToStage(actionExecContext, stage); } RoleGraph rg; http://git-wip-us.apache.org/repos/asf/ambari/blob/8def5a40/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelperImpl.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelperImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelperImpl.java index 6f407c9..11f578f 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelperImpl.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelperImpl.java @@ -1731,15 +1731,10 @@ public class KerberosHelperImpl implements KerberosHelper { Integer timeout) throws AmbariException { Stage stage = createNewStage(id, cluster, requestId, requestContext, clusterHostInfo, commandParams, hostParams); - stage.addServerActionCommand(actionClass.getName(), null, - Role.AMBARI_SERVER_ACTION, - RoleCommand.EXECUTE, - cluster.getClusterName(), - event, - commandParameters, - commandDetail, - ambariManagementController.findConfigurationTagsWithOverrides(cluster, null), - timeout, false); + stage.addServerActionCommand(actionClass.getName(), null, Role.AMBARI_SERVER_ACTION, + RoleCommand.EXECUTE, cluster.getClusterName(), event, commandParameters, commandDetail, + ambariManagementController.findConfigurationTagsWithOverrides(cluster, null), timeout, + false, false); return stage; } @@ -2216,7 +2211,8 @@ public class KerberosHelperImpl implements KerberosHelper { "SET_KEYTAB", requestResourceFilters, requestParams); - customCommandExecutionHelper.addExecutionCommandsToStage(actionExecContext, stage, requestParams, false); + customCommandExecutionHelper.addExecutionCommandsToStage(actionExecContext, stage, + requestParams); } RoleGraph roleGraph = roleGraphFactory.createNew(roleCommandOrder); @@ -2283,7 +2279,8 @@ public class KerberosHelperImpl implements KerberosHelper { "REMOVE_KEYTAB", requestResourceFilters, requestParams); - customCommandExecutionHelper.addExecutionCommandsToStage(actionExecContext, stage, requestParams, false); + customCommandExecutionHelper.addExecutionCommandsToStage(actionExecContext, stage, + requestParams); } } http://git-wip-us.apache.org/repos/asf/ambari/blob/8def5a40/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterStackVersionResourceProvider.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterStackVersionResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterStackVersionResourceProvider.java index a11a99b..4868ca7 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterStackVersionResourceProvider.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterStackVersionResourceProvider.java @@ -513,7 +513,7 @@ public class ClusterStackVersionResourceProvider extends AbstractControllerResou actionContext.setTimeout(Short.valueOf(configuration.getDefaultAgentTaskTimeout(true))); try { - actionExecutionHelper.get().addExecutionCommandsToStage(actionContext, stage, false); + actionExecutionHelper.get().addExecutionCommandsToStage(actionContext, stage); } catch (AmbariException e) { throw new SystemException("Can not modify stage", e); } http://git-wip-us.apache.org/repos/asf/ambari/blob/8def5a40/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/HostStackVersionResourceProvider.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/HostStackVersionResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/HostStackVersionResourceProvider.java index 1051056..a09edd0 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/HostStackVersionResourceProvider.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/HostStackVersionResourceProvider.java @@ -427,7 +427,7 @@ public class HostStackVersionResourceProvider extends AbstractControllerResource req.addStages(Collections.singletonList(stage)); try { - actionExecutionHelper.get().addExecutionCommandsToStage(actionContext, stage, false); + actionExecutionHelper.get().addExecutionCommandsToStage(actionContext, stage); } catch (AmbariException e) { throw new SystemException("Can not modify stage", e); } http://git-wip-us.apache.org/repos/asf/ambari/blob/8def5a40/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java index 770cc04..d087945 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java @@ -117,6 +117,16 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider protected static final String UPGRADE_REQUEST_STATUS = "Upgrade/request_status"; protected static final String UPGRADE_ABORT_REASON = "Upgrade/abort_reason"; + /** + * Skip slave/client component failures if the tasks are skippable. + */ + protected static final String UPGRADE_SKIP_FAILURES = "Upgrade/skip_failures"; + + /** + * Skip service check failures if the tasks are skippable. + */ + protected static final String UPGRADE_SKIP_SC_FAILURES = "Upgrade/skip_service_check_failures"; + /* * Lifted from RequestResourceProvider */ @@ -207,6 +217,8 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider PROPERTY_IDS.add(UPGRADE_FROM_VERSION); PROPERTY_IDS.add(UPGRADE_TO_VERSION); PROPERTY_IDS.add(UPGRADE_DIRECTION); + PROPERTY_IDS.add(UPGRADE_SKIP_FAILURES); + PROPERTY_IDS.add(UPGRADE_SKIP_SC_FAILURES); PROPERTY_IDS.add(REQUEST_CONTEXT_ID); PROPERTY_IDS.add(REQUEST_CREATE_TIME_ID); @@ -578,6 +590,22 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider } } + // optionally skip failures - this can be supplied on either the request or + // in the upgrade pack explicitely + boolean skipComponentFailures = false; + boolean skipServiceCheckFailures = false; + if (requestMap.containsKey(UPGRADE_SKIP_FAILURES)) { + skipComponentFailures = Boolean.parseBoolean((String) requestMap.get(UPGRADE_SKIP_FAILURES)); + } + + if (requestMap.containsKey(UPGRADE_SKIP_SC_FAILURES)) { + skipServiceCheckFailures = Boolean.parseBoolean( + (String) requestMap.get(UPGRADE_SKIP_SC_FAILURES)); + } + + ctx.setAutoSkipComponentFailures(skipComponentFailures); + ctx.setAutoSkipServiceCheckFailures(skipServiceCheckFailures); + List<UpgradeGroupHolder> groups = s_upgradeHelper.createSequence(pack, ctx); if (groups.isEmpty()) { @@ -772,7 +800,7 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider // Remove unused config-types from 'newConfigurationsByType' Iterator<String> iterator = newConfigurationsByType.keySet().iterator(); while (iterator.hasNext()) { - String configType = (String) iterator.next(); + String configType = iterator.next(); if (skipConfigTypes.contains(configType)) { LOG.info("RU: Removing configs for config-type {}", configType); iterator.remove(); @@ -936,6 +964,8 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider actionContext.setIgnoreMaintenance(true); actionContext.setTimeout(Short.valueOf(s_configuration.getDefaultAgentTaskTimeout(false))); + actionContext.setRetryAllowed(allowRetry); + actionContext.setAutoSkipFailures(context.isComponentFailureAutoSkipped()); ExecuteCommandJson jsons = s_commandExecutionHelper.get().getCommandJson(actionContext, cluster); @@ -955,7 +985,7 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider stage.setStageId(stageId); entity.setStageId(Long.valueOf(stageId)); - s_actionExecutionHelper.get().addExecutionCommandsToStage(actionContext, stage, allowRetry); + s_actionExecutionHelper.get().addExecutionCommandsToStage(actionContext, stage); // need to set meaningful text on the command for (Map<String, HostRoleCommand> map : stage.getHostRoleCommands().values()) { @@ -993,6 +1023,8 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider "RESTART", filters, restartCommandParams); actionContext.setTimeout(Short.valueOf(s_configuration.getDefaultAgentTaskTimeout(false))); actionContext.setIgnoreMaintenance(true); + actionContext.setRetryAllowed(allowRetry); + actionContext.setAutoSkipFailures(context.isComponentFailureAutoSkipped()); ExecuteCommandJson jsons = s_commandExecutionHelper.get().getCommandJson(actionContext, cluster); @@ -1008,14 +1040,14 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider if (0L == stageId) { stageId = 1L; } + stage.setStageId(stageId); entity.setStageId(Long.valueOf(stageId)); Map<String, String> requestParams = new HashMap<String, String>(); requestParams.put("command", "RESTART"); - s_commandExecutionHelper.get().addExecutionCommandsToStage(actionContext, stage, requestParams, - allowRetry); + s_commandExecutionHelper.get().addExecutionCommandsToStage(actionContext, stage, requestParams); request.addStages(Collections.singletonList(stage)); } @@ -1044,6 +1076,8 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider actionContext.setTimeout(Short.valueOf(s_configuration.getDefaultAgentTaskTimeout(false))); actionContext.setIgnoreMaintenance(true); + actionContext.setRetryAllowed(allowRetry); + actionContext.setAutoSkipFailures(context.isServiceCheckFailureAutoSkipped()); ExecuteCommandJson jsons = s_commandExecutionHelper.get().getCommandJson(actionContext, cluster); @@ -1064,8 +1098,7 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider entity.setStageId(Long.valueOf(stageId)); Map<String, String> requestParams = getNewParameterMap(); - s_commandExecutionHelper.get().addExecutionCommandsToStage(actionContext, stage, requestParams, - allowRetry); + s_commandExecutionHelper.get().addExecutionCommandsToStage(actionContext, stage, requestParams); request.addStages(Collections.singletonList(stage)); } @@ -1137,7 +1170,8 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider actionContext.setTimeout(Short.valueOf((short) -1)); actionContext.setIgnoreMaintenance(true); - + actionContext.setRetryAllowed(allowRetry); + actionContext.setAutoSkipFailures(context.isComponentFailureAutoSkipped()); ExecuteCommandJson jsons = s_commandExecutionHelper.get().getCommandJson(actionContext, cluster); @@ -1157,13 +1191,11 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider entity.setStageId(Long.valueOf(stageId)); stage.addServerActionCommand(task.getImplementationClass(), - getManagementController().getAuthName(), - Role.AMBARI_SERVER_ACTION, - RoleCommand.EXECUTE, + getManagementController().getAuthName(), Role.AMBARI_SERVER_ACTION, RoleCommand.EXECUTE, cluster.getClusterName(), - new ServiceComponentHostServerActionEvent(null, - System.currentTimeMillis()), - commandParams, itemDetail, null, Integer.valueOf(1200), allowRetry); + new ServiceComponentHostServerActionEvent(null, System.currentTimeMillis()), commandParams, + itemDetail, null, Integer.valueOf(1200), allowRetry, + context.isComponentFailureAutoSkipped()); request.addStages(Collections.singletonList(stage)); } http://git-wip-us.apache.org/repos/asf/ambari/blob/8def5a40/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/HostRoleCommandDAO.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/HostRoleCommandDAO.java b/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/HostRoleCommandDAO.java index 9c91656..06799a0 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/HostRoleCommandDAO.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/HostRoleCommandDAO.java @@ -62,7 +62,8 @@ public class HostRoleCommandDAO { "SUM(CASE WHEN hrc.status = :in_progress THEN 1 ELSE 0 END), " + "SUM(CASE WHEN hrc.status = :pending THEN 1 ELSE 0 END), " + "SUM(CASE WHEN hrc.status = :queued THEN 1 ELSE 0 END), " + - "SUM(CASE WHEN hrc.status = :timedout THEN 1 ELSE 0 END)" + + "SUM(CASE WHEN hrc.status = :timedout THEN 1 ELSE 0 END)," + + "SUM(CASE WHEN hrc.status = :skipped_failed THEN 1 ELSE 0 END)" + ") FROM HostRoleCommandEntity hrc " + " GROUP BY hrc.requestId, hrc.stageId HAVING hrc.requestId = :requestId", HostRoleCommandStatusSummaryDTO.class.getName()); @@ -434,6 +435,7 @@ public class HostRoleCommandDAO { query.setParameter("pending", HostRoleStatus.PENDING); query.setParameter("queued", HostRoleStatus.QUEUED); query.setParameter("timedout", HostRoleStatus.TIMEDOUT); + query.setParameter("skipped_failed", HostRoleStatus.SKIPPED_FAILED); Map<Long, HostRoleCommandStatusSummaryDTO> map = new HashMap<Long, HostRoleCommandStatusSummaryDTO>(); http://git-wip-us.apache.org/repos/asf/ambari/blob/8def5a40/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/HostRoleCommandStatusSummaryDTO.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/HostRoleCommandStatusSummaryDTO.java b/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/HostRoleCommandStatusSummaryDTO.java index 54ade92..549cc96 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/HostRoleCommandStatusSummaryDTO.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/HostRoleCommandStatusSummaryDTO.java @@ -53,7 +53,8 @@ public class HostRoleCommandStatusSummaryDTO { Number inProgress, Number pending, Number queued, - Number timedout) { + Number timedout, + Number skippedFailed) { m_stageId = Long.valueOf(null == stageId ? 0L : stageId.longValue()); if (null != skippable) { @@ -76,7 +77,7 @@ public class HostRoleCommandStatusSummaryDTO { put(HostRoleStatus.PENDING, pending); put(HostRoleStatus.QUEUED, queued); put(HostRoleStatus.TIMEDOUT, timedout); - + put(HostRoleStatus.SKIPPED_FAILED, skippedFailed); } @SuppressWarnings("boxing") @@ -158,7 +159,9 @@ public class HostRoleCommandStatusSummaryDTO { 0L, // inProgress, 0L, // pending, 0L, // queued, - 0L); // timedout + 0L, // timedout + 0L // skippedFailed + ); } /** @@ -217,6 +220,12 @@ public class HostRoleCommandStatusSummaryDTO { return this; } - + /** + * For testing, set the number of {@link HostRoleStatus#SKIPPED_FAILED} tasks + */ + public HostRoleCommandStatusSummaryDTO skippedFailed(int count) { + put(HostRoleStatus.SKIPPED_FAILED, count); + return this; + } } http://git-wip-us.apache.org/repos/asf/ambari/blob/8def5a40/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostRoleCommandEntity.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostRoleCommandEntity.java b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostRoleCommandEntity.java index d99da6d..ae78890 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostRoleCommandEntity.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostRoleCommandEntity.java @@ -141,6 +141,13 @@ public class HostRoleCommandEntity { @Column(name = "retry_allowed", nullable = false) private Integer retryAllowed = Integer.valueOf(0); + /** + * If the command fails and is skippable, then this will instruct the + * scheduler to skip the command. + */ + @Column(name = "auto_skip_on_failure", nullable = false) + private Integer autoSkipOnFailure = Integer.valueOf(0); + // This is really command type as well as name @Column(name = "role_command") @Enumerated(EnumType.STRING) @@ -346,6 +353,26 @@ public class HostRoleCommandEntity { retryAllowed = enabled ? 1 : 0; } + /** + * Gets whether commands which fail and are retryable are automatically + * skipped and marked with {@link HostRoleStatus#SKIPPED_FAILED}. + * + * @return + */ + public boolean isFailureAutoSkipped() { + return autoSkipOnFailure != 0; + } + + /** + * Sets whether commands which fail and are retryable are automatically + * skipped and marked with {@link HostRoleStatus#SKIPPED_FAILED}. + * + * @param skipFailures + */ + public void setAutoSkipOnFailure(boolean skipFailures) { + autoSkipOnFailure = skipFailures ? 1 : 0; + } + @Override public boolean equals(Object o) { if (this == o) { http://git-wip-us.apache.org/repos/asf/ambari/blob/8def5a40/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeContext.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeContext.java b/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeContext.java index 86dbccd..b10db9e 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeContext.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeContext.java @@ -56,6 +56,20 @@ public class UpgradeContext { private String m_downgradeFromVersion = null; /** + * {@code true} if slave/client component failures should be automatically + * skipped. This will only automatically skip the failure if the task is + * skippable to begin with. + */ + private boolean m_autoSkipComponentFailures = false; + + /** + * {@code true} if service check failures should be automatically skipped. + * This will only automatically skip the failure if the task is skippable to + * begin with. + */ + private boolean m_autoSkipServiceCheckFailures = false; + + /** * Constructor. * * @param resolver @@ -223,7 +237,7 @@ public class UpgradeContext { /** * This method returns the non-finalized version we are downgrading from. - * + * * @return version cluster is downgrading from */ public String getDowngradeFromVersion() { @@ -232,11 +246,53 @@ public class UpgradeContext { /** * Set the HDP stack version we are downgrading from. - * + * * @param downgradeFromVersion */ public void setDowngradeFromVersion(String downgradeFromVersion) { - this.m_downgradeFromVersion = downgradeFromVersion; + m_downgradeFromVersion = downgradeFromVersion; + } + + /** + * Gets whether skippable components that failed are automatically skipped. + * + * @return the skipComponentFailures + */ + public boolean isComponentFailureAutoSkipped() { + return m_autoSkipComponentFailures; + } + + /** + * Sets whether skippable components that failed are automatically skipped. + * + * @param skipComponentFailures + * {@code true} to automatically skip component failures which are + * marked as skippable. + */ + public void setAutoSkipComponentFailures(boolean autoSkipComponentFailures) { + m_autoSkipComponentFailures = autoSkipComponentFailures; + } + + /** + * Gets whether skippable service checks that failed are automatically + * skipped. + * + * @return the skipServiceCheckFailures + */ + public boolean isServiceCheckFailureAutoSkipped() { + return m_autoSkipServiceCheckFailures; + } + + /** + * Sets whether skippable service checks that failed are automatically + * skipped. + * + * @param autoSkipServiceCheckFailures + * {@code true} to automatically skip service check failures which + * are marked as being skippable. + */ + public void setAutoSkipServiceCheckFailures(boolean autoSkipServiceCheckFailures) { + m_autoSkipServiceCheckFailures = autoSkipServiceCheckFailures; } } http://git-wip-us.apache.org/repos/asf/ambari/blob/8def5a40/ambari-server/src/main/java/org/apache/ambari/server/topology/LogicalRequest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/LogicalRequest.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/LogicalRequest.java index b7f95cf..34f8262 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/LogicalRequest.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/LogicalRequest.java @@ -18,6 +18,16 @@ package org.apache.ambari.server.topology; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.TreeSet; +import java.util.concurrent.atomic.AtomicLong; + import org.apache.ambari.server.AmbariException; import org.apache.ambari.server.actionmanager.HostRoleCommand; import org.apache.ambari.server.actionmanager.HostRoleStatus; @@ -36,16 +46,6 @@ import org.apache.ambari.server.state.host.HostImpl; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.TreeSet; -import java.util.concurrent.atomic.AtomicLong; - /** * Logical Request implementation. */ @@ -246,6 +246,7 @@ public class LogicalRequest extends Request { int pending = 0; int queued = 0; int timedout = 0; + int skippedFailed = 0; //todo: where does this logic belong? for (HostRoleCommandEntity task : stage.getHostRoleCommands()) { @@ -282,13 +283,19 @@ public class LogicalRequest extends Request { case TIMEDOUT: timedout += 1; break; + case SKIPPED_FAILED: + skippedFailed += 1; + break; default: System.out.println("Unexpected status when creating stage summaries: " + taskStatus); } } - HostRoleCommandStatusSummaryDTO stageSummary = new HostRoleCommandStatusSummaryDTO(stage.isSkippable() ? 1 : 0, 0, 0, - stage.getStageId(), aborted, completed, failed, holding, holdingFailed, holdingTimedout, inProgress, pending, queued, timedout); + HostRoleCommandStatusSummaryDTO stageSummary = new HostRoleCommandStatusSummaryDTO( + stage.isSkippable() ? 1 : 0, 0, 0, stage.getStageId(), aborted, completed, failed, + holding, holdingFailed, holdingTimedout, inProgress, pending, queued, timedout, + skippedFailed); + summaryMap.put(stage.getStageId(), stageSummary); } return summaryMap; http://git-wip-us.apache.org/repos/asf/ambari/blob/8def5a40/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog212.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog212.java b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog212.java index 02df181..0b3d75c 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog212.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog212.java @@ -18,10 +18,14 @@ package org.apache.ambari.server.upgrade; -import com.google.inject.Inject; -import com.google.inject.Injector; +import java.sql.SQLException; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.regex.Matcher; + import org.apache.ambari.server.AmbariException; -import org.apache.ambari.server.configuration.Configuration.DatabaseType; import org.apache.ambari.server.controller.AmbariManagementController; import org.apache.ambari.server.orm.DBAccessor.DBColumnInfo; import org.apache.ambari.server.orm.dao.DaoUtils; @@ -32,19 +36,9 @@ import org.apache.ambari.server.state.StackId; import org.apache.ambari.server.utils.VersionUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.jdbc.support.JdbcUtils; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.sql.Statement; -import java.text.MessageFormat; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.atomic.AtomicLong; -import java.util.regex.Matcher; +import com.google.inject.Inject; +import com.google.inject.Injector; /** @@ -57,6 +51,9 @@ public class UpgradeCatalog212 extends AbstractUpgradeCatalog { private static final String HBASE_SITE = "hbase-site"; private static final String CLUSTER_ENV = "cluster-env"; + private static final String HOST_ROLE_COMMAND_TABLE = "host_role_command"; + private static final String HOST_ROLE_COMMAND_SKIP_COLUMN = "auto_skip_on_failure"; + /** * Logger. */ @@ -104,6 +101,7 @@ public class UpgradeCatalog212 extends AbstractUpgradeCatalog { */ @Override protected void executeDDLUpdates() throws AmbariException, SQLException { + executeHostRoleCommandDDLUpdates(); } /** @@ -220,4 +218,14 @@ public class UpgradeCatalog212 extends AbstractUpgradeCatalog { return hiveEnvContent.replaceAll(oldHeapSizeRegex, Matcher.quoteReplacement(newAuxJarPath)); } + /** + * DDL changes for {@link #HOST_ROLE_COMMAND_TABLE}. + * + * @throws AmbariException + * @throws SQLException + */ + private void executeHostRoleCommandDDLUpdates() throws AmbariException, SQLException { + dbAccessor.addColumn(HOST_ROLE_COMMAND_TABLE, + new DBColumnInfo(HOST_ROLE_COMMAND_SKIP_COLUMN, Integer.class, 1, 0, false)); + } } http://git-wip-us.apache.org/repos/asf/ambari/blob/8def5a40/ambari-server/src/main/java/org/apache/ambari/server/utils/StageUtils.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/utils/StageUtils.java b/ambari-server/src/main/java/org/apache/ambari/server/utils/StageUtils.java index 3da0fe2..e51cfb2 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/utils/StageUtils.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/utils/StageUtils.java @@ -201,8 +201,8 @@ public class StageUtils { s.setStageId(stageId); long now = System.currentTimeMillis(); s.addHostRoleExecutionCommand(hostname, Role.NAMENODE, RoleCommand.INSTALL, - new ServiceComponentHostInstallEvent("NAMENODE", hostname, now, "HDP-1.2.0"), - "cluster1", "HDFS", false); + new ServiceComponentHostInstallEvent("NAMENODE", hostname, now, "HDP-1.2.0"), "cluster1", + "HDFS", false, false); ExecutionCommand execCmd = s.getExecutionCommandWrapper(hostname, "NAMENODE").getExecutionCommand(); execCmd.setRequestAndStage(s.getRequestId(), s.getStageId()); http://git-wip-us.apache.org/repos/asf/ambari/blob/8def5a40/ambari-server/src/main/resources/Ambari-DDL-MySQL-CREATE.sql ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/Ambari-DDL-MySQL-CREATE.sql b/ambari-server/src/main/resources/Ambari-DDL-MySQL-CREATE.sql index 265e42e..70623d8 100644 --- a/ambari-server/src/main/resources/Ambari-DDL-MySQL-CREATE.sql +++ b/ambari-server/src/main/resources/Ambari-DDL-MySQL-CREATE.sql @@ -235,6 +235,7 @@ CREATE TABLE host_role_command ( start_time BIGINT NOT NULL, end_time BIGINT, status VARCHAR(255), + auto_skip_on_failure SMALLINT DEFAULT 0 NOT NULL, std_error LONGBLOB, std_out LONGBLOB, output_log VARCHAR(255) NULL, http://git-wip-us.apache.org/repos/asf/ambari/blob/8def5a40/ambari-server/src/main/resources/Ambari-DDL-Oracle-CREATE.sql ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/Ambari-DDL-Oracle-CREATE.sql b/ambari-server/src/main/resources/Ambari-DDL-Oracle-CREATE.sql index 0053837..0cedeb7 100644 --- a/ambari-server/src/main/resources/Ambari-DDL-Oracle-CREATE.sql +++ b/ambari-server/src/main/resources/Ambari-DDL-Oracle-CREATE.sql @@ -225,6 +225,7 @@ CREATE TABLE host_role_command ( start_time NUMBER(19) NOT NULL, end_time NUMBER(19), status VARCHAR2(255) NULL, + auto_skip_on_failure NUMBER(1) DEFAULT 0 NOT NULL, std_error BLOB NULL, std_out BLOB NULL, output_log VARCHAR2(255) NULL,
