Repository: ambari Updated Branches: refs/heads/trunk 3cd575b86 -> 814594e52
AMBARI-8811 - Rolling Upgrade: inconsistent statuses (tbeerbower) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/814594e5 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/814594e5 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/814594e5 Branch: refs/heads/trunk Commit: 814594e524071da471c0d41a8a407e5f7eca96c1 Parents: 3cd575b Author: tbeerbower <[email protected]> Authored: Fri Dec 19 10:56:57 2014 -0500 Committer: tbeerbower <[email protected]> Committed: Fri Dec 19 10:57:05 2014 -0500 ---------------------------------------------------------------------- .../internal/RequestResourceProvider.java | 88 +++++++------------- .../internal/StageResourceProvider.java | 57 ++++++++----- .../internal/UpgradeGroupResourceProvider.java | 65 ++++++++------- .../internal/StageResourceProviderTest.java | 57 +++++++++++++ 4 files changed, 162 insertions(+), 105 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/814594e5/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/RequestResourceProvider.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/RequestResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/RequestResourceProvider.java index 549a20d..3c4524a 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/RequestResourceProvider.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/RequestResourceProvider.java @@ -44,6 +44,7 @@ import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; +import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Set; @@ -162,8 +163,6 @@ public class RequestResourceProvider extends AbstractControllerResourceProvider Integer maxResults = (maxResultsRaw == null ? null : Integer.parseInt(maxResultsRaw)); Boolean ascOrder = (ascOrderRaw == null ? null : Boolean.parseBoolean(ascOrderRaw)); - Set<Map<String, Object>> propertyMaps = new HashSet<Map<String,Object>>(); - if (null == predicate) { resources.addAll( getRequestResources(null, null, null, maxResults, ascOrder, requestedIds)); @@ -215,15 +214,16 @@ public class RequestResourceProvider extends AbstractControllerResourceProvider } // There should be only one request with this id (or no request at all) org.apache.ambari.server.actionmanager.Request internalRequest = internalRequests.get(0); - // Validate update request (check constraints on state value and presense of abort reason) + // Validate update request (check constraints on state value and presence of abort reason) if (updateRequest.getAbortReason() == null || updateRequest.getAbortReason().isEmpty()) { throw new IllegalArgumentException("Abort reason can not be empty."); } - HostRoleStatus internalRequestStatus = calculateSummaryStatus( - calculateRequestStatusCounters(internalRequest), - internalRequest.getCommands().size() - ); + List<HostRoleCommand> commands = internalRequest.getCommands(); + HostRoleStatus internalRequestStatus = + StageResourceProvider.calculateSummaryStatus( + StageResourceProvider.calculateTaskStatusCounts(getHostRoleStatuses(commands)), commands.size()); + if (updateRequest.getStatus() != HostRoleStatus.ABORTED) { throw new IllegalArgumentException( String.format("%s is wrong value. The only allowed value " + @@ -240,8 +240,7 @@ public class RequestResourceProvider extends AbstractControllerResourceProvider } // Perform update Iterator<RequestRequest> reqIterator = requests.iterator(); - for (int i = 0; i < targets.size(); i++) { - org.apache.ambari.server.actionmanager.Request target = targets.get(i); + for (org.apache.ambari.server.actionmanager.Request target : targets) { String reason = reqIterator.next().getAbortReason(); amc.getActionManager().cancelRequest(target.getRequestId(), reason); } @@ -307,7 +306,8 @@ public class RequestResourceProvider extends AbstractControllerResourceProvider } List<RequestResourceFilter> resourceFilterList = null; - Set<Map<String, Object>> resourceFilters = null; + Set<Map<String, Object>> resourceFilters; + Object resourceFilterObj = propertyMap.get(REQUEST_RESOURCE_FILTER_ID); if (resourceFilterObj != null && resourceFilterObj instanceof HashSet) { resourceFilters = (HashSet<Map<String, Object>>) resourceFilterObj; @@ -463,15 +463,17 @@ public class RequestResourceProvider extends AbstractControllerResourceProvider } else { setResourceProperty(resource, REQUEST_SOURCE_SCHEDULE, null, requestedPropertyIds); } - int taskCount = request.getCommands().size(); - Map<HostRoleStatus, Integer> hostRoleStatusCounters = calculateRequestStatusCounters(request); - HostRoleStatus requestStatus = calculateSummaryStatus(hostRoleStatusCounters, taskCount); + Collection<HostRoleCommand> commands = request.getCommands(); + + int taskCount = commands.size(); - double progressPercent = - ((hostRoleStatusCounters.get(HostRoleStatus.QUEUED) * 0.09 + - hostRoleStatusCounters.get(HostRoleStatus.IN_PROGRESS) * 0.35 + - hostRoleStatusCounters.get(HostRoleStatus.COMPLETED)) / taskCount) * 100.0; + Map<HostRoleStatus, Integer> hostRoleStatusCounters = + StageResourceProvider.calculateTaskStatusCounts(getHostRoleStatuses(commands)); + + HostRoleStatus requestStatus = StageResourceProvider.calculateSummaryStatus(hostRoleStatusCounters, taskCount); + + double progressPercent = StageResourceProvider.calculateProgressPercent(hostRoleStatusCounters, taskCount); setResourceProperty(resource, REQUEST_STATUS_PROPERTY_ID, requestStatus.toString(), requestedPropertyIds); setResourceProperty(resource, REQUEST_TASK_CNT_ID, taskCount, requestedPropertyIds); @@ -491,48 +493,18 @@ public class RequestResourceProvider extends AbstractControllerResourceProvider } /** - * Returns counts of tasks that are in various states. + * Get a collection of statuses from the given collection of tasks. + * + * @param tasks the tasks + * + * @return a collection of statuses */ - private Map<HostRoleStatus, Integer> calculateRequestStatusCounters(org.apache.ambari.server.actionmanager.Request request) { - List<HostRoleCommand> commands = request.getCommands(); - Map<HostRoleStatus, Integer> counters = new HashMap<HostRoleStatus, Integer>(); - int totalTasks = commands.size(); - // initialize - for (HostRoleStatus hostRoleStatus : HostRoleStatus.values()) { - counters.put(hostRoleStatus, 0); - } - // calculate counts - for (HostRoleCommand hostRoleCommand : commands) { - HostRoleStatus status = hostRoleCommand.getStatus(); - if (status.isCompletedState() && - status != HostRoleStatus.COMPLETED ) { // we don't want to count twice - // Increase total number of completed tasks; - counters.put(HostRoleStatus.COMPLETED, counters.get(HostRoleStatus.COMPLETED) + 1); - } - // Increment counter for particular status - counters.put(status, counters.get(status) + 1); - } - // We overwrite the value to have the sum converged - counters.put(HostRoleStatus.IN_PROGRESS, - totalTasks - counters.get(HostRoleStatus.COMPLETED)- - counters.get(HostRoleStatus.QUEUED)- - counters.get(HostRoleStatus.PENDING)); - return counters; - } + private static Collection<HostRoleStatus> getHostRoleStatuses(Collection<HostRoleCommand> tasks) { + Collection<HostRoleStatus> hostRoleStatuses = new LinkedList<HostRoleStatus>(); - /** - * @param counters counts of tasks that are in various states. - * @param totalTasks total number of tasks in request - * @return summary request status based on statuses of tasks in different - * states. - */ - static HostRoleStatus calculateSummaryStatus(Map<HostRoleStatus, Integer> counters, int totalTasks) { - return counters.get(HostRoleStatus.FAILED) > 0 ? HostRoleStatus.FAILED : - // TODO (dlysnichenko): maybe change order of FAILED and ABORTED? - counters.get(HostRoleStatus.ABORTED) > 0 ? HostRoleStatus.ABORTED : - counters.get(HostRoleStatus.TIMEDOUT) > 0 ? HostRoleStatus.TIMEDOUT : - counters.get(HostRoleStatus.IN_PROGRESS) > 0 ? HostRoleStatus.IN_PROGRESS : - counters.get(HostRoleStatus.COMPLETED) == totalTasks ? HostRoleStatus.COMPLETED : - HostRoleStatus.PENDING; + for (HostRoleCommand hostRoleCommand : tasks) { + hostRoleStatuses.add(hostRoleCommand.getStatus()); + } + return hostRoleStatuses; } } http://git-wip-us.apache.org/repos/asf/ambari/blob/814594e5/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StageResourceProvider.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StageResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StageResourceProvider.java index f2ea76d..c174a9c 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StageResourceProvider.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StageResourceProvider.java @@ -47,6 +47,7 @@ import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; import java.util.LinkedHashSet; +import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Set; @@ -262,7 +263,7 @@ public class StageResourceProvider extends AbstractResourceProvider implements E private static void updateStageStatus(StageEntity entity, HostRoleStatus desiredStatus) { Collection<HostRoleCommandEntity> tasks = entity.getHostRoleCommands(); - Map<HostRoleStatus, Integer> taskStatusCounts = calculateTaskStatusCounts(tasks); + Map<HostRoleStatus, Integer> taskStatusCounts = calculateTaskStatusCounts(getHostRoleStatuses(tasks)); HostRoleStatus currentStatus = calculateSummaryStatus(taskStatusCounts, tasks.size()); @@ -331,7 +332,7 @@ public class StageResourceProvider extends AbstractResourceProvider implements E int taskCount = tasks.size(); - Map<HostRoleStatus, Integer> taskStatusCounts = calculateTaskStatusCounts(tasks); + Map<HostRoleStatus, Integer> taskStatusCounts = calculateTaskStatusCounts(getHostRoleStatuses(tasks)); setResourceProperty(resource, STAGE_PROGRESS_PERCENT, calculateProgressPercent(taskStatusCounts, taskCount), requestedIds); @@ -342,28 +343,31 @@ public class StageResourceProvider extends AbstractResourceProvider implements E } /** - * Calculate the percent complete for the stage based on its tasks. + * Calculate the percent complete based on the given status counts. * - * @param counters counts of tasks that are in various states - * @param totalTasks total number of tasks in request + * @param counters counts of resources that are in various states + * @param total total number of resources in request * * @return the percent complete for the stage */ - private double calculateProgressPercent(Map<HostRoleStatus, Integer> counters, double totalTasks) { - return ((counters.get(HostRoleStatus.QUEUED) * 0.09 + - counters.get(HostRoleStatus.IN_PROGRESS) * 0.35 + - counters.get(HostRoleStatus.COMPLETED)) / totalTasks) * 100.0; + protected static double calculateProgressPercent(Map<HostRoleStatus, Integer> counters, double total) { + return ((counters.get(HostRoleStatus.QUEUED) * 0.09 + + counters.get(HostRoleStatus.IN_PROGRESS) * 0.35 + + counters.get(HostRoleStatus.HOLDING) * 0.35 + + counters.get(HostRoleStatus.HOLDING_FAILED) * 0.35 + + counters.get(HostRoleStatus.HOLDING_TIMEDOUT) * 0.35 + + counters.get(HostRoleStatus.COMPLETED)) / total) * 100.0; } /** - * Calculate the stage status based on the status of its tasks. + * Calculate an overall status based on the given status counts. * - * @param counters counts of tasks that are in various states - * @param totalTasks total number of tasks in request + * @param counters counts of resources that are in various states + * @param total total number of resources in request * * @return summary request status based on statuses of tasks in different states. */ - private static HostRoleStatus calculateSummaryStatus(Map<HostRoleStatus, Integer> counters, int totalTasks) { + protected static HostRoleStatus calculateSummaryStatus(Map<HostRoleStatus, Integer> counters, int total) { return counters.get(HostRoleStatus.HOLDING) > 0 ? HostRoleStatus.HOLDING : counters.get(HostRoleStatus.HOLDING_FAILED) > 0 ? HostRoleStatus.HOLDING_FAILED : counters.get(HostRoleStatus.HOLDING_TIMEDOUT) > 0 ? HostRoleStatus.HOLDING_TIMEDOUT : @@ -371,25 +375,40 @@ public class StageResourceProvider extends AbstractResourceProvider implements E counters.get(HostRoleStatus.ABORTED) > 0 ? HostRoleStatus.ABORTED : counters.get(HostRoleStatus.TIMEDOUT) > 0 ? HostRoleStatus.TIMEDOUT : counters.get(HostRoleStatus.IN_PROGRESS) > 0 ? HostRoleStatus.IN_PROGRESS : - counters.get(HostRoleStatus.COMPLETED) == totalTasks ? HostRoleStatus.COMPLETED : HostRoleStatus.PENDING; + counters.get(HostRoleStatus.COMPLETED) == total ? HostRoleStatus.COMPLETED : HostRoleStatus.PENDING; + } + + /** + * Get a collection of statuses from the given collection of task entities. + * + * @param tasks the task entities + * + * @return a collection of statuses + */ + private static Collection<HostRoleStatus> getHostRoleStatuses(Collection<HostRoleCommandEntity> tasks) { + Collection<HostRoleStatus> hostRoleStatuses = new LinkedList<HostRoleStatus>(); + + for (HostRoleCommandEntity hostRoleCommand : tasks) { + hostRoleStatuses.add(hostRoleCommand.getStatus()); + } + return hostRoleStatuses; } /** * Returns counts of tasks that are in various states. * - * @param tasks the collection of tasks + * @param hostRoleStatuses the collection of tasks * * @return a map of counts of tasks keyed by the task status */ - private static Map<HostRoleStatus, Integer> calculateTaskStatusCounts(Collection<HostRoleCommandEntity> tasks) { + protected static Map<HostRoleStatus, Integer> calculateTaskStatusCounts(Collection<HostRoleStatus> hostRoleStatuses) { Map<HostRoleStatus, Integer> counters = new HashMap<HostRoleStatus, Integer>(); // initialize for (HostRoleStatus hostRoleStatus : HostRoleStatus.values()) { counters.put(hostRoleStatus, 0); } // calculate counts - for (HostRoleCommandEntity hostRoleCommand : tasks) { - HostRoleStatus status = hostRoleCommand.getStatus(); + for (HostRoleStatus status : hostRoleStatuses) { // count tasks where isCompletedState() == true as COMPLETED // but don't count tasks with COMPLETED status twice if (status.isCompletedState() && status != HostRoleStatus.COMPLETED) { @@ -402,7 +421,7 @@ public class StageResourceProvider extends AbstractResourceProvider implements E // We overwrite the value to have the sum converged counters.put(HostRoleStatus.IN_PROGRESS, - tasks.size() - + hostRoleStatuses.size() - counters.get(HostRoleStatus.COMPLETED) - counters.get(HostRoleStatus.QUEUED) - counters.get(HostRoleStatus.PENDING)); http://git-wip-us.apache.org/repos/asf/ambari/blob/814594e5/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeGroupResourceProvider.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeGroupResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeGroupResourceProvider.java index 46dcd1e..47f6237 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeGroupResourceProvider.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeGroupResourceProvider.java @@ -22,6 +22,7 @@ import java.util.Arrays; import java.util.Collection; import java.util.HashMap; import java.util.HashSet; +import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Set; @@ -86,7 +87,7 @@ public class UpgradeGroupResourceProvider extends AbstractControllerResourceProv /** * Constructor. * - * @param controller + * @param controller the controller */ UpgradeGroupResourceProvider(AmbariManagementController controller) { super(PROPERTY_IDS, KEY_PROPERTY_IDS, controller); @@ -176,47 +177,55 @@ public class UpgradeGroupResourceProvider extends AbstractControllerResourceProv return resource; } - /** * Aggregates status and percent complete for stages and puts the results on the upgrade group * - * @param upgradeGroup the resource representing an upgrade group - * @param stages the collection of resources representing stages - * @param requestedIds the ids for the request + * @param upgradeGroup the resource representing an upgrade group + * @param stages the collection of resources representing stages + * @param requestedIds the ids for the request */ private void aggregate(Resource upgradeGroup, Collection<Resource> stages, Set<String> requestedIds) { - Double total = new Double(0d); - Integer count = new Integer(0); + Map<HostRoleStatus, Integer> counters = + StageResourceProvider.calculateTaskStatusCounts(getHostRoleStatuses(stages)); - Map<HostRoleStatus, Integer> counters = new HashMap<HostRoleStatus, Integer>(); - for (HostRoleStatus hostRoleStatus : HostRoleStatus.values()) { - counters.put(hostRoleStatus, new Integer(0)); - } + setResourceProperty(upgradeGroup, UPGRADE_GROUP_STATUS, + StageResourceProvider.calculateSummaryStatus(counters, stages.size()), requestedIds); + setResourceProperty(upgradeGroup, UPGRADE_GROUP_PROGRESS_PERCENT, + StageResourceProvider.calculateProgressPercent(counters, stages.size()), requestedIds); + } - for (Resource stage : stages) { - String status = (String) stage.getPropertyValue(StageResourceProvider.STAGE_STATUS); - Double percent = (Double) stage.getPropertyValue(StageResourceProvider.STAGE_PROGRESS_PERCENT); + /** + * Get a collection of statuses from the given collection of stage resources. + * + * @param stageResources the stage resources + * + * @return a collection of statuses + */ + private static Collection<HostRoleStatus> getHostRoleStatuses(Collection<Resource> stageResources) { + Collection<HostRoleStatus> hostRoleStatuses = new LinkedList<HostRoleStatus>(); - if (null != percent) { - total += percent; - count++; - } + for (Resource stage : stageResources) { + HostRoleStatus status = getStatus(stage); - if (null != status) { - HostRoleStatus hrs = HostRoleStatus.valueOf(status); - counters.put(hrs, counters.get(hrs) + 1); + if (status != null) { + hostRoleStatuses.add(status); } } + return hostRoleStatuses; + } - if (count > 0) { - setResourceProperty(upgradeGroup, UPGRADE_GROUP_PROGRESS_PERCENT, total/count, requestedIds); - } - - HostRoleStatus summary = RequestResourceProvider.calculateSummaryStatus(counters, counters.size()); - setResourceProperty(upgradeGroup, UPGRADE_GROUP_STATUS, summary, requestedIds); + /** + * Get the status of the given stage resource. + * + * @param stageResource the resource + * + * @return the stage status + */ + private static HostRoleStatus getStatus(Resource stageResource) { + String status = (String) stageResource.getPropertyValue(StageResourceProvider.STAGE_STATUS); + return status == null ? null : HostRoleStatus.valueOf(status); } - } http://git-wip-us.apache.org/repos/asf/ambari/blob/814594e5/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/StageResourceProviderTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/StageResourceProviderTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/StageResourceProviderTest.java index c97de8a..4c47e94 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/StageResourceProviderTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/StageResourceProviderTest.java @@ -178,6 +178,63 @@ public class StageResourceProviderTest { verify(dao, clusters, cluster); } + @Test + public void testCalculateTaskStatusCounts() { + + Collection<HostRoleStatus> hostRoleStatuses = new LinkedList<HostRoleStatus>(); + + hostRoleStatuses.add(HostRoleStatus.PENDING); + hostRoleStatuses.add(HostRoleStatus.QUEUED); + hostRoleStatuses.add(HostRoleStatus.HOLDING); + hostRoleStatuses.add(HostRoleStatus.HOLDING_FAILED); + hostRoleStatuses.add(HostRoleStatus.HOLDING_TIMEDOUT); + hostRoleStatuses.add(HostRoleStatus.IN_PROGRESS); + hostRoleStatuses.add(HostRoleStatus.IN_PROGRESS); + hostRoleStatuses.add(HostRoleStatus.COMPLETED); + hostRoleStatuses.add(HostRoleStatus.COMPLETED); + hostRoleStatuses.add(HostRoleStatus.COMPLETED); + hostRoleStatuses.add(HostRoleStatus.COMPLETED); + hostRoleStatuses.add(HostRoleStatus.FAILED); + hostRoleStatuses.add(HostRoleStatus.TIMEDOUT); + hostRoleStatuses.add(HostRoleStatus.ABORTED); + + Map<HostRoleStatus, Integer> counts = StageResourceProvider.calculateTaskStatusCounts(hostRoleStatuses); + + assertEquals(1L, (long) counts.get(HostRoleStatus.PENDING)); + assertEquals(1L, (long) counts.get(HostRoleStatus.QUEUED)); + assertEquals(1L, (long) counts.get(HostRoleStatus.HOLDING)); + assertEquals(1L, (long) counts.get(HostRoleStatus.HOLDING_FAILED)); + assertEquals(1L, (long) counts.get(HostRoleStatus.HOLDING_TIMEDOUT)); + assertEquals(5L, (long) counts.get(HostRoleStatus.IN_PROGRESS)); + assertEquals(7L, (long) counts.get(HostRoleStatus.COMPLETED)); + assertEquals(1L, (long) counts.get(HostRoleStatus.FAILED)); + assertEquals(1L, (long) counts.get(HostRoleStatus.TIMEDOUT)); + assertEquals(1L, (long) counts.get(HostRoleStatus.ABORTED)); + } + + @Test + public void testCalculateProgressPercent() { + + Collection<HostRoleStatus> hostRoleStatuses = new LinkedList<HostRoleStatus>(); + + hostRoleStatuses.add(HostRoleStatus.PENDING); + hostRoleStatuses.add(HostRoleStatus.QUEUED); + hostRoleStatuses.add(HostRoleStatus.HOLDING); + hostRoleStatuses.add(HostRoleStatus.IN_PROGRESS); + hostRoleStatuses.add(HostRoleStatus.IN_PROGRESS); + hostRoleStatuses.add(HostRoleStatus.COMPLETED); + hostRoleStatuses.add(HostRoleStatus.COMPLETED); + hostRoleStatuses.add(HostRoleStatus.FAILED); + hostRoleStatuses.add(HostRoleStatus.TIMEDOUT); + hostRoleStatuses.add(HostRoleStatus.ABORTED); + + Map<HostRoleStatus, Integer> counts = StageResourceProvider.calculateTaskStatusCounts(hostRoleStatuses); + + Double percent = StageResourceProvider.calculateProgressPercent(counts, counts.size()); + + assertEquals(Double.valueOf(64.9), percent); + } + private List<StageEntity> getStageEntities() { StageEntity stage = new StageEntity();
