SLIDER-965 RoleStatus and AppState move to using LongGauges to store numbers in
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/ac98d826 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/ac98d826 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/ac98d826 Branch: refs/heads/feature/SLIDER-82-pass-3.1 Commit: ac98d826e0e0bc21590f8b25aa4366707b054de2 Parents: 5b7f6dd Author: Steve Loughran <[email protected]> Authored: Fri Nov 6 21:17:25 2015 +0000 Committer: Steve Loughran <[email protected]> Committed: Fri Nov 6 21:17:25 2015 +0000 ---------------------------------------------------------------------- .../slider/api/proto/RestTypeMarshalling.java | 1 - .../slider/api/types/ComponentInformation.java | 1 - .../server/appmaster/management/LongGauge.java | 42 ++++++++++--- .../slider/server/appmaster/state/AppState.java | 42 +++++-------- .../server/appmaster/state/RoleStatus.java | 62 ++++++++++++++++---- 5 files changed, 101 insertions(+), 47 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ac98d826/slider-core/src/main/java/org/apache/slider/api/proto/RestTypeMarshalling.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/api/proto/RestTypeMarshalling.java b/slider-core/src/main/java/org/apache/slider/api/proto/RestTypeMarshalling.java index 115405c..b7985e6 100644 --- a/slider-core/src/main/java/org/apache/slider/api/proto/RestTypeMarshalling.java +++ b/slider-core/src/main/java/org/apache/slider/api/proto/RestTypeMarshalling.java @@ -85,7 +85,6 @@ public class RestTypeMarshalling { info.failureMessage = wire.getFailureMessage(); } info.pendingAntiAffineRequestCount = wire.getPendingAntiAffineRequestCount(); - info.pendingAntiAffineRequest = info.pendingAntiAffineRequestCount > 0; return info; } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ac98d826/slider-core/src/main/java/org/apache/slider/api/types/ComponentInformation.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/api/types/ComponentInformation.java b/slider-core/src/main/java/org/apache/slider/api/types/ComponentInformation.java index 3b4b8bd..9d8a4ee 100644 --- a/slider-core/src/main/java/org/apache/slider/api/types/ComponentInformation.java +++ b/slider-core/src/main/java/org/apache/slider/api/types/ComponentInformation.java @@ -52,7 +52,6 @@ public class ComponentInformation { public int requested; public int failed, started, startFailed, completed, totalRequested; public int nodeFailed, failedRecently, preempted; - public boolean pendingAntiAffineRequest; public int pendingAntiAffineRequestCount; public String failureMessage; http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ac98d826/slider-core/src/main/java/org/apache/slider/server/appmaster/management/LongGauge.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/management/LongGauge.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/management/LongGauge.java index 72a8805..ac9ac0e 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/management/LongGauge.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/management/LongGauge.java @@ -25,7 +25,9 @@ import java.util.concurrent.atomic.AtomicLong; /** * This is a {@link AtomicLong} which acts as a metrics gauge: its state can be exposed as - * a management value. + * a metrics. + * It also exposes some of the same method names as the Codahale Counter class, so that + * it's easy to swap in. * */ public class LongGauge extends AtomicLong implements Metric, Gauge<Long> { @@ -45,22 +47,46 @@ public class LongGauge extends AtomicLong implements Metric, Gauge<Long> { this(0); } - + /** + * Get the value as a metric + * @return current value + */ @Override public Long getValue() { return get(); } + public Long getCount() { + return get(); + } + + /** + * {@code ++} + */ + public void inc() { + incrementAndGet(); + } + /** + * {@code --} + */ + public void dec() { + decrementAndGet(); + } + /** - * Decrement to the floor of 0. - * There's checks to stop more than one thread being in this method at the time, but - * that doesn't stop other operations on the value + * Decrement to the floor of 0. Operations in parallel may cause confusion here, + * but it will still never go below zero * @param delta delta * @return the current value */ - public synchronized long decToFloor(long delta) { - long newval = Math.max(0L, get() - delta); - set(newval); + public long decToFloor(long delta) { + long l = get(); + long r = l - delta; + if (r < 0) { + r = 0; + } + // if this fails, the decrement has been lost + compareAndSet(l, r); return get(); } } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ac98d826/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java index c46177a..f6fe474 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java @@ -19,6 +19,7 @@ package org.apache.slider.server.appmaster.state; import com.codahale.metrics.Counter; +import com.codahale.metrics.Metric; import com.codahale.metrics.MetricRegistry; import com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.conf.Configuration; @@ -62,6 +63,7 @@ import org.apache.slider.core.persist.AggregateConfSerDeser; import org.apache.slider.core.persist.ConfTreeSerDeser; import org.apache.slider.providers.PlacementPolicy; import org.apache.slider.providers.ProviderRole; +import org.apache.slider.server.appmaster.management.LongGauge; import org.apache.slider.server.appmaster.management.MetricsAndMonitoring; import org.apache.slider.server.appmaster.management.MetricsConstants; import org.apache.slider.server.appmaster.operations.AbstractRMOperation; @@ -194,33 +196,33 @@ public class AppState { /** * Counter for completed containers ( complete denotes successful or failed ) */ - private final Counter completedContainerCount = new Counter(); + private final LongGauge completedContainerCount = new LongGauge(); /** * Count of failed containers */ - private final Counter failedContainerCount = new Counter(); + private final LongGauge failedContainerCount = new LongGauge(); /** * # of started containers */ - private final Counter startedContainers = new Counter(); + private final LongGauge startedContainers = new LongGauge(); /** * # of containers that failed to start */ - private final Counter startFailedContainerCount = new Counter(); + private final LongGauge startFailedContainerCount = new LongGauge(); /** * Track the number of surplus containers received and discarded */ - private final Counter surplusContainers = new Counter(); + private final LongGauge surplusContainers = new LongGauge(); /** * Track the number of requested Containers */ - private final Counter outstandingContainerRequests = new Counter(); + private final LongGauge outstandingContainerRequests = new LongGauge(); /** * Map of requested nodes. This records the command used to start it, @@ -1211,22 +1213,10 @@ public class AppState { } /** - * dec requested count of a role - * <p> - * Also updates application state counters. - * @param role role to decrement - */ - protected synchronized void decrementRequestCount(RoleStatus role) { - role.decRequested(); - } - - /** * Inc #of outstanding requests. */ private void incOutstandingContainerRequests() { - synchronized (outstandingContainerRequests) { - outstandingContainerRequests.inc(); - } + outstandingContainerRequests.inc(); } /** @@ -1700,7 +1690,7 @@ public class AppState { */ public ApplicationLivenessInformation getApplicationLivenessInformation() { ApplicationLivenessInformation li = new ApplicationLivenessInformation(); - int outstanding = (int) outstandingContainerRequests.getCount(); + int outstanding = outstandingContainerRequests.intValue(); li.requestsOutstanding = outstanding; li.allRequestsSatisfied = outstanding <= 0; return li; @@ -1716,15 +1706,15 @@ public class AppState { sliderstats.put(StatusKeys.STATISTICS_CONTAINERS_LIVE, liveNodes.size()); sliderstats.put(StatusKeys.STATISTICS_CONTAINERS_COMPLETED, - (int)completedContainerCount.getCount()); + completedContainerCount.intValue()); sliderstats.put(StatusKeys.STATISTICS_CONTAINERS_FAILED, - (int)failedContainerCount.getCount()); + failedContainerCount.intValue()); sliderstats.put(StatusKeys.STATISTICS_CONTAINERS_STARTED, - (int)startedContainers.getCount()); + startedContainers.intValue()); sliderstats.put(StatusKeys.STATISTICS_CONTAINERS_START_FAILED, - (int) startFailedContainerCount.getCount()); + startFailedContainerCount.intValue()); sliderstats.put(StatusKeys.STATISTICS_CONTAINERS_SURPLUS, - (int)surplusContainers.getCount()); + surplusContainers.intValue()); sliderstats.put(StatusKeys.STATISTICS_CONTAINERS_UNKNOWN_COMPLETED, completionOfUnknownContainerEvent.get()); return sliderstats; @@ -2085,7 +2075,7 @@ public class AppState { final RoleStatus role = lookupRoleStatus(container); //dec requested count - decrementRequestCount(role); + role.decRequested(); //inc allocated count -this may need to be dropped in a moment, // but us needed to update the logic below http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ac98d826/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java index 4197c4f..cba963c 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java @@ -18,6 +18,8 @@ package org.apache.slider.server.appmaster.state; +import com.google.common.base.Preconditions; +import org.apache.hadoop.yarn.api.records.Resource; import org.apache.slider.api.types.ComponentInformation; import org.apache.slider.providers.PlacementPolicy; import org.apache.slider.providers.ProviderRole; @@ -60,17 +62,17 @@ public final class RoleStatus implements Cloneable { private final LongGauge failedRecently = new LongGauge(0); private final LongGauge limitsExceeded = new LongGauge(0); - /** flag set to true if there is an outstanding anti-affine request */ - private final AtomicBoolean pendingAARequest = new AtomicBoolean(false); + /** resource requirements */ + private Resource resourceRequirements; /** * Number of AA requests queued. These should be reduced first on a * flex down. */ - private int pendingAntiAffineRequestCount = 0; + private final LongGauge pendingAntiAffineRequests = new LongGauge(0); /** any pending AA request */ - public OutstandingRequest outstandingAArequest = null; + private OutstandingRequest outstandingAArequest = null; private String failureMessage = ""; @@ -163,9 +165,8 @@ public final class RoleStatus implements Cloneable { return requested.incrementAndGet(); } - - public long cancel(long count) { - return requested.decToFloor(count); + public void cancel(long count) { + requested.decToFloor(count); } public void decRequested() { @@ -204,6 +205,10 @@ public final class RoleStatus implements Cloneable { return limitsExceeded.get(); } + public long incPendingAntiAffineRequests(long v) { + return pendingAntiAffineRequests.addAndGet(v); + } + /** * Note that a role failed, text will * be used in any diagnostics if an exception @@ -285,6 +290,22 @@ public final class RoleStatus implements Cloneable { return nodeFailed.get(); } + public long getPendingAntiAffineRequests() { + return pendingAntiAffineRequests.get(); + } + + public void setPendingAntiAffineRequests(long pendingAntiAffineRequests) { + this.pendingAntiAffineRequests.set(pendingAntiAffineRequests); + } + + public OutstandingRequest getOutstandingAArequest() { + return outstandingAArequest; + } + + public void setOutstandingAArequest(OutstandingRequest outstandingAArequest) { + this.outstandingAArequest = outstandingAArequest; + } + /** * Get the number of roles we are short of. * nodes released are ignored. @@ -321,7 +342,7 @@ public final class RoleStatus implements Cloneable { ", actual=" + actual + ", requested=" + requested + ", releasing=" + releasing + - ", pendingAntiAffineRequestCount=" + pendingAntiAffineRequestCount + + ", pendingAntiAffineRequestCount=" + pendingAntiAffineRequests + ", failed=" + failed + ", failed recently=" + failedRecently.get() + ", node failed=" + nodeFailed.get() + @@ -376,8 +397,7 @@ public final class RoleStatus implements Cloneable { info.failedRecently = failedRecently.intValue(); info.nodeFailed = nodeFailed.intValue(); info.preempted = preempted.intValue(); - info.pendingAntiAffineRequest = pendingAARequest.get(); - info.pendingAntiAffineRequestCount = pendingAntiAffineRequestCount; + info.pendingAntiAffineRequestCount = pendingAntiAffineRequests.intValue(); return info; } @@ -389,6 +409,14 @@ public final class RoleStatus implements Cloneable { return providerRole.labelExpression; } + public Resource getResourceRequirements() { + return resourceRequirements; + } + + public void setResourceRequirements(Resource resourceRequirements) { + this.resourceRequirements = resourceRequirements; + } + /** * Compare two role status entries by name */ @@ -410,5 +438,17 @@ public final class RoleStatus implements Cloneable { return (o1.getKey() < o2.getKey() ? -1 : (o1.getKey() == o2.getKey() ? 0 : 1)); } } - + + /** + * Given a resource, set its requirements to those this role needs + * @param resource resource to configure + * @return the resource + */ + public Resource copyResourceRequirements(Resource resource) { + Preconditions.checkNotNull(resourceRequirements, + "Role resource requirements have not been set"); + resource.setMemory(resourceRequirements.getMemory()); + resource.setVirtualCores(resourceRequirements.getVirtualCores()); + return resource; + } }
