SLIDER-768: renaming attributes to emphasise "outstanding"; mock tests to validate; ClusterDescription exporting
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/b75581b5 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/b75581b5 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/b75581b5 Branch: refs/heads/develop Commit: b75581b526ad621a2a8432b4fcbcd86afb078e35 Parents: 7f2e0c2 Author: Steve Loughran <[email protected]> Authored: Mon Feb 2 14:37:34 2015 +0000 Committer: Steve Loughran <[email protected]> Committed: Mon Feb 2 14:37:34 2015 +0000 ---------------------------------------------------------------------- .../apache/slider/api/ClusterDescription.java | 8 +++- .../types/ApplicationLivenessInformation.java | 4 +- .../appmaster/management/MetricsConstants.java | 27 +++++++++++ .../slider/server/appmaster/state/AppState.java | 48 +++++++++++++++----- .../rest/RestAPIClientTestDelegates.groovy | 2 +- .../appstate/TestMockAppStateFlexing.groovy | 25 +++++++++- 6 files changed, 97 insertions(+), 17 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b75581b5/slider-core/src/main/java/org/apache/slider/api/ClusterDescription.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/api/ClusterDescription.java b/slider-core/src/main/java/org/apache/slider/api/ClusterDescription.java index 8bcff0e..7db7e7f 100644 --- a/slider-core/src/main/java/org/apache/slider/api/ClusterDescription.java +++ b/slider-core/src/main/java/org/apache/slider/api/ClusterDescription.java @@ -23,6 +23,7 @@ import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.slider.api.types.ApplicationLivenessInformation; import org.apache.slider.common.tools.SliderUtils; import org.apache.slider.core.exceptions.BadConfigException; import org.apache.slider.providers.SliderProviderFactory; @@ -204,7 +205,12 @@ public class ClusterDescription implements Cloneable { * Status information */ public Map<String, Object> status; - + + /** + * Liveness information; the same as returned + * on the <code>live/liveness/</code> URL + */ + public ApplicationLivenessInformation liveness; /** * Creator. http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b75581b5/slider-core/src/main/java/org/apache/slider/api/types/ApplicationLivenessInformation.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/api/types/ApplicationLivenessInformation.java b/slider-core/src/main/java/org/apache/slider/api/types/ApplicationLivenessInformation.java index d4d4278..4dd2cb7 100644 --- a/slider-core/src/main/java/org/apache/slider/api/types/ApplicationLivenessInformation.java +++ b/slider-core/src/main/java/org/apache/slider/api/types/ApplicationLivenessInformation.java @@ -31,14 +31,14 @@ import org.codehaus.jackson.map.annotate.JsonSerialize; @JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL) public class ApplicationLivenessInformation { public boolean allRequestsSatisfied; - public int requested; + public int requestsOutstanding; @Override public String toString() { final StringBuilder sb = new StringBuilder("ApplicationLivenessInformation{"); sb.append("allRequestsSatisfied=").append(allRequestsSatisfied); - sb.append(", requested=").append(requested); + sb.append(", requestsOutstanding=").append(requestsOutstanding); sb.append('}'); return sb.toString(); } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b75581b5/slider-core/src/main/java/org/apache/slider/server/appmaster/management/MetricsConstants.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/management/MetricsConstants.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/management/MetricsConstants.java new file mode 100644 index 0000000..e55cf60 --- /dev/null +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/management/MetricsConstants.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.server.appmaster.management; + +/** + * Constants used in slider for metrics registration and lookup + */ +public class MetricsConstants { + public static final String CONTAINERS_OUTSTANDING_REQUESTS = + "containers.outstanding-requests"; +} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b75581b5/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java index 97a22b3..5a8b38b 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java @@ -19,6 +19,7 @@ package org.apache.slider.server.appmaster.state; import com.codahale.metrics.Counter; +import com.codahale.metrics.MetricRegistry; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import org.apache.hadoop.conf.Configuration; @@ -67,6 +68,7 @@ import org.apache.slider.core.persist.ConfTreeSerDeser; import org.apache.slider.providers.PlacementPolicy; import org.apache.slider.providers.ProviderRole; import org.apache.slider.server.appmaster.management.MetricsAndMonitoring; +import org.apache.slider.server.appmaster.management.MetricsConstants; import org.apache.slider.server.appmaster.operations.AbstractRMOperation; import org.apache.slider.server.appmaster.operations.CancelRequestOperation; import org.apache.slider.server.appmaster.operations.ContainerReleaseOperation; @@ -224,8 +226,7 @@ public class AppState { /** * Track the number of requested Containers */ - private final Counter requestedContainerCount = new Counter(); - + private final Counter outstandingContainerRequests = new Counter(); /** * Map of requested nodes. This records the command used to start it, @@ -295,9 +296,17 @@ public class AppState { * @param recordFactory factory for YARN records * @param metricsAndMonitoring metrics and monitoring services */ - public AppState(AbstractRecordFactory recordFactory, MetricsAndMonitoring metricsAndMonitoring) { + public AppState(AbstractRecordFactory recordFactory, + MetricsAndMonitoring metricsAndMonitoring) { this.recordFactory = recordFactory; - this.metricsAndMonitoring = metricsAndMonitoring; + this.metricsAndMonitoring = metricsAndMonitoring; + + // register any metrics + MetricRegistry metrics = metricsAndMonitoring.getMetrics(); + metrics.register( + MetricRegistry.name(AppState.class, + MetricsConstants.CONTAINERS_OUTSTANDING_REQUESTS), + outstandingContainerRequests); } public int getFailedCountainerCount() { @@ -381,8 +390,15 @@ public class AppState { } /** - * Get the dynamcally created view of the cluster status - * @return + * Get the current view of the cluster status. + * <p> + * Calls to {@link #refreshClusterStatus()} trigger a + * refresh of this field. + * <p> + * This is read-only + * to the extent that changes here do not trigger updates in the + * application state. + * @return the cluster status */ public synchronized ClusterDescription getClusterStatus() { return clusterStatus; @@ -617,7 +633,7 @@ public class AppState { BadConfigException { String priOpt = component.getMandatoryOption(ResourceKeys.COMPONENT_PRIORITY); int priority = SliderUtils.parseAndValidate("value of " + name + " " + - ResourceKeys.COMPONENT_PRIORITY, + ResourceKeys.COMPONENT_PRIORITY, priOpt, 0, 1, -1); String placementOpt = component.getOption( ResourceKeys.COMPONENT_PLACEMENT_POLICY, @@ -1205,7 +1221,7 @@ public class AppState { */ protected void incrementRequestCount(RoleStatus role) { role.incRequested(); - requestedContainerCount.inc(); + outstandingContainerRequests.inc(); } /** @@ -1216,7 +1232,7 @@ public class AppState { */ protected void decrementRequestCount(RoleStatus role) { role.decRequested(); - requestedContainerCount.dec(); + outstandingContainerRequests.dec(); } @@ -1603,13 +1619,16 @@ public class AppState { return percentage; } + /** + * Update the cluster description with the current application state + */ public void refreshClusterStatus() { refreshClusterStatus(null); } /** - * Update the cluster description with anything interesting + * Update the cluster description with the current application state * @param providerStatus status from the provider for the cluster info section */ public synchronized ClusterDescription refreshClusterStatus(Map<String, String> providerStatus) { @@ -1654,8 +1673,13 @@ public class AppState { cd.statistics.put(rolename, stats); } + Map<String, Integer> sliderstats = getLiveStatistics(); cd.statistics.put(SliderKeys.COMPONENT_AM, sliderstats); + + // liveness + cd.liveness = getApplicationLivenessInformation(); + return cd; } @@ -1665,8 +1689,8 @@ public class AppState { */ public ApplicationLivenessInformation getApplicationLivenessInformation() { ApplicationLivenessInformation li = new ApplicationLivenessInformation(); - int outstanding = (int) requestedContainerCount.getCount(); - li.requested = outstanding; + int outstanding = (int) outstandingContainerRequests.getCount(); + li.requestsOutstanding = outstanding; li.allRequestsSatisfied = outstanding == 0; return li; } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b75581b5/slider-core/src/test/groovy/org/apache/slider/agent/rest/RestAPIClientTestDelegates.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/rest/RestAPIClientTestDelegates.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/rest/RestAPIClientTestDelegates.groovy index 3762d5d..029cb70 100644 --- a/slider-core/src/test/groovy/org/apache/slider/agent/rest/RestAPIClientTestDelegates.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/agent/rest/RestAPIClientTestDelegates.groovy @@ -248,6 +248,6 @@ class RestAPIClientTestDelegates extends AbstractRestTestDelegate { describe "Liveness:\n$liveness" assert liveness.allRequestsSatisfied - assert !liveness.requested + assert !liveness.requestsOutstanding } } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b75581b5/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexing.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexing.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexing.groovy index 1db500b..dcb862d 100644 --- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexing.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexing.groovy @@ -39,9 +39,19 @@ class TestMockAppStateFlexing extends BaseMockAppStateTest implements MockRoles @Test public void testFlexDuringLaunchPhase() throws Throwable { + + // ask for one instance of role0 role0Status.desired = 1 List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes() + + // at this point there's now one request in the list + assert 1 == ops.size() + // and in a liveness check, one outstanding + def liveness = appState.applicationLivenessInformation + assert 1 == liveness.requestsOutstanding + assert !liveness.allRequestsSatisfied + List<Container> allocations = engine.execute(ops) List<ContainerAssignment> assignments = []; List<AbstractRMOperation> releases = [] @@ -54,6 +64,10 @@ class TestMockAppStateFlexing extends BaseMockAppStateTest implements MockRoles ops = appState.reviewRequestAndReleaseNodes() assert ops.empty + liveness = appState.applicationLivenessInformation + assert 0 == liveness.requestsOutstanding + assert liveness.allRequestsSatisfied + //now this is the start point. appState.containerStartSubmitted(target, ri); @@ -69,8 +83,17 @@ class TestMockAppStateFlexing extends BaseMockAppStateTest implements MockRoles List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes() assert !ops.empty + + // second scan will find the first run outstanding, so not re-issue + // any more container requests List<AbstractRMOperation> ops2 = appState.reviewRequestAndReleaseNodes() assert ops2.empty + + // and in a liveness check, one outstanding + def liveness = appState.applicationLivenessInformation + assert 1 == liveness.requestsOutstanding + assert !liveness.allRequestsSatisfied + } @@ -104,7 +127,7 @@ class TestMockAppStateFlexing extends BaseMockAppStateTest implements MockRoles // now shrink again - role0Status.desired = r0 = 1 + role0Status.desired = 1 completionResults = [] instances = createStartAndStopNodes(completionResults) assert instances.size() == 0
