Repository: incubator-slider Updated Branches: refs/heads/develop 566b3f66f -> 0d2a35bb6
SLIDER-741 expose raw/resolved views of app configurations Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/0d2a35bb Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/0d2a35bb Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/0d2a35bb Branch: refs/heads/develop Commit: 0d2a35bb637155bc9f88e662966a1a2246ed0da0 Parents: 566b3f6 Author: Steve Loughran <[email protected]> Authored: Tue Jan 6 19:32:57 2015 +0000 Committer: Steve Loughran <[email protected]> Committed: Tue Jan 6 19:32:57 2015 +0000 ---------------------------------------------------------------------- .../core/persist/AggregateConfSerDeser.java | 55 +++++++ .../server/appmaster/SliderAppMaster.java | 4 +- .../slider/server/appmaster/state/AppState.java | 87 ++++++++--- .../appmaster/state/ProviderAppState.java | 5 + .../state/StateAccessForProviders.java | 10 ++ .../server/appmaster/web/rest/RestPaths.java | 5 +- .../rest/application/ApplicationResource.java | 143 +++++++++++++++---- .../resources/AggregateModelRefresher.java | 47 ++++++ .../application/resources/AppconfRefresher.java | 55 +++++++ .../application/resources/CachedContent.java | 6 +- .../application/resources/ContentCache.java | 3 +- .../resources/LiveComponentsRefresher.java | 3 +- .../resources/LiveContainersRefresher.java | 3 +- .../resources/LiveResourcesRefresher.java | 2 +- .../resources/ResourceRefresher.java | 2 +- .../resources/ResourceSnapshotRefresher.java | 2 +- .../standalone/TestStandaloneAgentWeb.groovy | 64 ++++++++- .../TestMockAppStateAppRestIntegration.groovy | 2 +- .../apache/slider/test/SliderTestUtils.groovy | 17 +++ 19 files changed, 445 insertions(+), 70 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0d2a35bb/slider-core/src/main/java/org/apache/slider/core/persist/AggregateConfSerDeser.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/core/persist/AggregateConfSerDeser.java b/slider-core/src/main/java/org/apache/slider/core/persist/AggregateConfSerDeser.java new file mode 100644 index 0000000..90537b6 --- /dev/null +++ b/slider-core/src/main/java/org/apache/slider/core/persist/AggregateConfSerDeser.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.core.persist; + +import org.apache.slider.core.conf.AggregateConf; +import org.codehaus.jackson.JsonGenerationException; +import org.codehaus.jackson.JsonParseException; +import org.codehaus.jackson.map.JsonMappingException; + +import java.io.IOException; + +/** + * Conf tree to JSON binding + */ +public class AggregateConfSerDeser extends JsonSerDeser<AggregateConf> { + public AggregateConfSerDeser() { + super(AggregateConf.class); + } + + + private static final AggregateConfSerDeser + staticinstance = new AggregateConfSerDeser(); + + /** + * Convert a tree instance to a JSON string -sync access to a shared ser/deser + * object instance + * @param instance object to convert + * @return a JSON string description + * @throws JsonParseException parse problems + * @throws JsonMappingException O/J mapping problems + */ + public static String toString(AggregateConf instance) throws IOException, + JsonGenerationException, + JsonMappingException { + synchronized (staticinstance) { + return staticinstance.toJson(instance); + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0d2a35bb/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java index c8764c9..0dd6c59 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java @@ -596,8 +596,8 @@ public class SliderAppMaster extends AbstractSliderLaunchedService instanceDefinition.getAppConfOperations().set( KEY_SECURITY_ENABLED, securityEnabled); - // triggers resolution and snapshotting in agent - appState.updateInstanceDefinition(instanceDefinition); + // triggers resolution and snapshotting for agent + appState.setInitialInstanceDefinition(instanceDefinition); File confDir = getLocalConfDir(); if (!confDir.exists() || !confDir.isDirectory()) { http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0d2a35bb/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java index 9ae20a5..f381cde 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java @@ -59,6 +59,8 @@ import org.apache.slider.core.exceptions.ErrorStrings; import org.apache.slider.core.exceptions.NoSuchNodeException; import org.apache.slider.core.exceptions.SliderInternalStateException; import org.apache.slider.core.exceptions.TriggerClusterTeardownException; +import org.apache.slider.core.persist.AggregateConfSerDeser; +import org.apache.slider.core.persist.ConfTreeSerDeser; import org.apache.slider.providers.PlacementPolicy; import org.apache.slider.providers.ProviderRole; import org.apache.slider.server.appmaster.management.MetricsAndMonitoring; @@ -109,16 +111,30 @@ public class AppState { /** * The definition of the instance. Flexing updates the resources section - This is used as a synchronization point on activities that update - the CD, and also to update some of the structures that - feed in to the CD + * This is used as a synchronization point on activities that update + * the CD, and also to update some of the structures that + * feed in to the CD */ private AggregateConf instanceDefinition; - + + /** + * Time the instance definition snapshots were created + */ private long snapshotTime; + + /** + * Snapshot of the instance definition. This is fully + * resolved. + */ private AggregateConf instanceDefinitionSnapshot; /** + * Snapshot of the raw instance definition; unresolved and + * without any patch of an AM into it. + */ + private AggregateConf unresolvedInstanceDefinition; + + /** * snapshot of resources as of last update time */ private ConfTreeOperations resourcesSnapshot; @@ -379,13 +395,18 @@ public class AppState { * * Important: this is for early binding and must not be used after the build * operation is complete. - * @param definition + * @param definition initial definition * @throws BadConfigException */ - public synchronized void updateInstanceDefinition(AggregateConf definition) throws - BadConfigException, - IOException { - this.instanceDefinition = definition; + public synchronized void setInitialInstanceDefinition(AggregateConf definition) + throws BadConfigException, IOException { + log.debug("Setting initial instance definition"); + // snapshot the definition + AggregateConfSerDeser serDeser = new AggregateConfSerDeser(); + + unresolvedInstanceDefinition = serDeser.fromInstance(definition); + + this.instanceDefinition = serDeser.fromInstance(definition); onInstanceDefinitionUpdated(); } @@ -453,6 +474,10 @@ public class AppState { return instanceDefinitionSnapshot; } + public AggregateConf getUnresolvedInstanceDefinition() { + return unresolvedInstanceDefinition; + } + /** * Build up the application state * @param instanceDefinition definition of the applicatin instance @@ -462,8 +487,8 @@ public class AppState { * @param fs filesystem * @param historyDir directory containing history files * @param liveContainers list of live containers supplied on an AM restart - * @param applicationInfo - * @param releaseSelector + * @param applicationInfo app info to retain for web views + * @param releaseSelector selector of containers to release */ public synchronized void buildInstance(AggregateConf instanceDefinition, Configuration appmasterConfig, @@ -478,6 +503,7 @@ public class AppState { Preconditions.checkArgument(instanceDefinition != null); Preconditions.checkArgument(releaseSelector != null); + log.debug("Building application state"); this.publishedProviderConf = publishedProviderConf; this.applicationInfo = applicationInfo != null ? applicationInfo : new HashMap<String, String>(); @@ -496,7 +522,7 @@ public class AppState { // set the cluster specification (once its dependency the client properties // is out the way - updateInstanceDefinition(instanceDefinition); + setInitialInstanceDefinition(instanceDefinition); //build the initial role list for (ProviderRole providerRole : providerRoles) { @@ -544,8 +570,8 @@ public class AppState { rebuildModelFromRestart(liveContainers); // any am config options to pick up - logServerURL = appmasterConfig.get(YarnConfiguration.YARN_LOG_SERVER_URL, ""); + //mark as live applicationLive = true; } @@ -607,13 +633,26 @@ public class AppState { /** * Actions to perform when an instance definition is updated - * Currently: resolve the configuration - * updated the cluster spec derivative + * Currently: + * <ol> + * <li> + * resolve the configuration + * </li> + * <li> + * update the cluster spec derivative + * </li> + * </ol> + * * @throws BadConfigException */ private synchronized void onInstanceDefinitionUpdated() throws BadConfigException, IOException { + log.debug("Instance definition updated"); + //note the time + snapshotTime = now(); + + // resolve references if not already done instanceDefinition.resolve(); // force in the AM desired state values @@ -624,8 +663,7 @@ public class AppState { SliderKeys.COMPONENT_AM, ResourceKeys.COMPONENT_INSTANCES, "1"); } - //note the time - snapshotTime = now(); + //snapshot all three sectons resourcesSnapshot = ConfTreeOperations.fromInstance(instanceDefinition.getResources()); @@ -660,11 +698,16 @@ public class AppState { public synchronized List<ProviderRole> updateResourceDefinitions(ConfTree resources) throws BadConfigException, IOException { log.debug("Updating resources to {}", resources); - - instanceDefinition.setResources(resources); + // snapshot the (possibly unresolved) values + ConfTreeSerDeser serDeser = new ConfTreeSerDeser(); + unresolvedInstanceDefinition.setResources( + serDeser.fromInstance(resources)); + // assign another copy under the instance definition for resolving + // and then driving application size + instanceDefinition.setResources(serDeser.fromInstance(resources)); onInstanceDefinitionUpdated(); - //propagate the role table + // propagate the role table Map<String, Map<String, String>> updated = resources.components; getClusterStatus().roles = SliderUtils.deepClone(updated); getClusterStatus().updateTime = now(); @@ -952,14 +995,12 @@ public class AppState { return findNodeInCollection(containerId, nodes); } - - /** * Iterate through a collection of role instances to find one with a * specific (string) container ID * @param containerId container ID as a string * @param nodes collection - * @return + * @return the found node * @throws NoSuchNodeException if there was no match */ private RoleInstance findNodeInCollection(String containerId, http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0d2a35bb/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ProviderAppState.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ProviderAppState.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ProviderAppState.java index d5a041b..43ff52f 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ProviderAppState.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ProviderAppState.java @@ -154,6 +154,11 @@ public class ProviderAppState implements StateAccessForProviders { public AggregateConf getInstanceDefinitionSnapshot() { return appState.getInstanceDefinitionSnapshot(); } + + @Override + public AggregateConf getUnresolvedInstanceDefinition() { + return appState.getUnresolvedInstanceDefinition(); + } @Override public RoleStatus lookupRoleStatus(int key) { http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0d2a35bb/slider-core/src/main/java/org/apache/slider/server/appmaster/state/StateAccessForProviders.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/StateAccessForProviders.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/StateAccessForProviders.java index 75076ed..2cbe6b1 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/StateAccessForProviders.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/StateAccessForProviders.java @@ -117,9 +117,19 @@ public interface StateAccessForProviders { long getSnapshotTime(); + /** + * Get a snapshot of the entire aggregate configuration + * @return the aggregate configuration + */ AggregateConf getInstanceDefinitionSnapshot(); /** + * Get the desired/unresolved value + * @return unresolved + */ + AggregateConf getUnresolvedInstanceDefinition(); + + /** * Look up a role from its key -or fail * * @param key key to resolve http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0d2a35bb/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/RestPaths.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/RestPaths.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/RestPaths.java index 011ec3a..f637190 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/RestPaths.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/RestPaths.java @@ -134,10 +134,11 @@ public class RestPaths { public static final String LIVE_RESOURCES = "/live/resources"; public static final String LIVE_CONTAINERS = "/live/containers"; public static final String LIVE_COMPONENTS = "/live/components"; - public static final String MODEL_DESIRED = "/model/desired"; + public static final String MODEL = "/model"; + public static final String MODEL_DESIRED = MODEL +"/desired"; public static final String MODEL_DESIRED_APPCONF = MODEL_DESIRED +"/appconf"; public static final String MODEL_DESIRED_RESOURCES = MODEL_DESIRED +"/resources"; - public static final String MODEL_RESOLVED = "/model/desired"; + public static final String MODEL_RESOLVED = "/model/resolved"; public static final String MODEL_RESOLVED_APPCONF = MODEL_RESOLVED +"/appconf"; public static final String MODEL_RESOLVED_RESOURCES = MODEL_RESOLVED +"/resources"; public static final String MODEL_INTERNAL = "/model/internal"; http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0d2a35bb/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/ApplicationResource.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/ApplicationResource.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/ApplicationResource.java index 9419765..48540d4 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/ApplicationResource.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/ApplicationResource.java @@ -19,11 +19,11 @@ package org.apache.slider.server.appmaster.web.rest.application; import com.google.common.collect.Lists; -import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.webapp.NotFoundException; import org.apache.slider.api.types.SerializedComponentInformation; import org.apache.slider.api.types.SerializedContainerInformation; +import org.apache.slider.core.conf.AggregateConf; import org.apache.slider.core.conf.ConfTree; import org.apache.slider.core.exceptions.NoSuchNodeException; import org.apache.slider.server.appmaster.state.RoleInstance; @@ -31,7 +31,10 @@ import org.apache.slider.server.appmaster.state.RoleStatus; import org.apache.slider.server.appmaster.state.StateAccessForProviders; import org.apache.slider.server.appmaster.web.WebAppApi; import org.apache.slider.server.appmaster.web.rest.AbstractSliderResource; -import org.apache.slider.server.appmaster.web.rest.RestPaths; +import static org.apache.slider.server.appmaster.web.rest.RestPaths.*; + +import org.apache.slider.server.appmaster.web.rest.application.resources.AggregateModelRefresher; +import org.apache.slider.server.appmaster.web.rest.application.resources.AppconfRefresher; import org.apache.slider.server.appmaster.web.rest.application.resources.CachedContent; import org.apache.slider.server.appmaster.web.rest.application.resources.LiveContainersRefresher; import org.apache.slider.server.appmaster.web.rest.application.resources.ContentCache; @@ -63,23 +66,46 @@ public class ApplicationResource extends AbstractSliderResource { "nodes", "statistics", "internal"); + public static final List<String> ROOT_ENTRIES = toJsonList("model", "live", "actions"); + + public static final List<String> MODEL_ENTRIES = + toJsonList("desired", "resolved"); + private final ContentCache cache = new ContentCache(); private final StateAccessForProviders state; public ApplicationResource(WebAppApi slider) { super(slider); state = slider.getAppState(); - cache.put(RestPaths.LIVE_RESOURCES, + cache.put(LIVE_RESOURCES, new CachedContent<ConfTree>(LIFESPAN, new LiveResourcesRefresher(state))); - cache.put(RestPaths.LIVE_CONTAINERS, + cache.put(LIVE_CONTAINERS, new CachedContent<Map<String, SerializedContainerInformation>>(LIFESPAN, new LiveContainersRefresher(state))); - cache.put(RestPaths.LIVE_COMPONENTS, + cache.put(LIVE_COMPONENTS, new CachedContent<Map<String, SerializedComponentInformation>> (LIFESPAN, new LiveComponentsRefresher(state))); + cache.put(MODEL_DESIRED, + new CachedContent<AggregateConf>(LIFESPAN, + new AggregateModelRefresher(state, false))); + cache.put(MODEL_RESOLVED, + new CachedContent<AggregateConf>(LIFESPAN, + new AggregateModelRefresher(state, true))); + cache.put(MODEL_RESOLVED_APPCONF, + new CachedContent<ConfTree>(LIFESPAN, + new AppconfRefresher(state, false, false))); + cache.put(MODEL_RESOLVED_RESOURCES, + new CachedContent<ConfTree>(LIFESPAN, + new AppconfRefresher(state, false, true))); + cache.put(MODEL_DESIRED_APPCONF, + new CachedContent<ConfTree>(LIFESPAN, + new AppconfRefresher(state, true, false))); + cache.put(MODEL_DESIRED_RESOURCES, + new CachedContent<ConfTree>(LIFESPAN, + new AppconfRefresher(state, true, true))); } /** @@ -98,45 +124,87 @@ public class ApplicationResource extends AbstractSliderResource { return ROOT_ENTRIES; } + /** + * Enum model values: desired and resolved + * @return the desired and resolved model + */ @GET - @Path("/model") + @Path(MODEL) @Produces({MediaType.APPLICATION_JSON}) public List<String> getModel() { - return toJsonList("desired", "resolved"); + return MODEL_ENTRIES; } @GET - @Path("/live") + @Path(MODEL_DESIRED) + @Produces({MediaType.APPLICATION_JSON}) + public AggregateConf getModelDesired() { + return lookupAggregateConf(MODEL_DESIRED); + } + + @GET + @Path(MODEL_DESIRED_APPCONF) + @Produces({MediaType.APPLICATION_JSON}) + public ConfTree getModelDesiredAppconf() { + return lookupConfTree(MODEL_DESIRED_APPCONF); + } + + @GET + @Path(MODEL_DESIRED_RESOURCES) + @Produces({MediaType.APPLICATION_JSON}) + public ConfTree getModelDesiredResources() { + return lookupConfTree(MODEL_DESIRED_RESOURCES); + } + + @GET + @Path(MODEL_RESOLVED) + @Produces({MediaType.APPLICATION_JSON}) + public AggregateConf getModelResolved() { + return lookupAggregateConf(MODEL_RESOLVED); + } + + @GET + @Path(MODEL_RESOLVED_APPCONF) + @Produces({MediaType.APPLICATION_JSON}) + public ConfTree getModelResolvedAppconf() { + return lookupConfTree(MODEL_RESOLVED_APPCONF); + } + + @GET + @Path(MODEL_RESOLVED_RESOURCES) + @Produces({MediaType.APPLICATION_JSON}) + public ConfTree getModelResolvedResources() { + return lookupConfTree(MODEL_RESOLVED_RESOURCES); + } + + @GET + @Path(LIVE) @Produces({MediaType.APPLICATION_JSON}) public List<String> getLive() { return LIVE_ENTRIES; } @GET - @Path(RestPaths.LIVE_RESOURCES) + @Path(LIVE_RESOURCES) @Produces({MediaType.APPLICATION_JSON}) public Object getLiveResources() { - try { - return cache.lookup(RestPaths.LIVE_RESOURCES); - } catch (Exception e) { - throw buildException(RestPaths.LIVE_RESOURCES, e); - } + return lookupConfTree(LIVE_RESOURCES); } @GET - @Path(RestPaths.LIVE_CONTAINERS) + @Path(LIVE_CONTAINERS) @Produces({MediaType.APPLICATION_JSON}) public Map<String, SerializedContainerInformation> getLiveContainers() { try { return (Map<String, SerializedContainerInformation>)cache.lookup( - RestPaths.LIVE_CONTAINERS); + LIVE_CONTAINERS); } catch (Exception e) { - throw buildException(RestPaths.LIVE_CONTAINERS, e); + throw buildException(LIVE_CONTAINERS, e); } } @GET - @Path(RestPaths.LIVE_CONTAINERS + "/{containerId}") + @Path(LIVE_CONTAINERS + "/{containerId}") @Produces({MediaType.APPLICATION_JSON}) public SerializedContainerInformation getLiveContainer( @PathParam("containerId") String containerId) { @@ -146,24 +214,24 @@ public class ApplicationResource extends AbstractSliderResource { } catch (NoSuchNodeException e) { throw new NotFoundException("Unknown container: " + containerId); } catch (Exception e) { - throw buildException(RestPaths.LIVE_CONTAINERS + "/"+ containerId, e); + throw buildException(LIVE_CONTAINERS + "/"+ containerId, e); } } @GET - @Path(RestPaths.LIVE_COMPONENTS) + @Path(LIVE_COMPONENTS) @Produces({MediaType.APPLICATION_JSON}) public Map<String, SerializedComponentInformation> getLiveComponents() { try { return (Map<String, SerializedComponentInformation>) cache.lookup( - RestPaths.LIVE_COMPONENTS); + LIVE_COMPONENTS); } catch (Exception e) { - throw buildException(RestPaths.LIVE_COMPONENTS, e); + throw buildException(LIVE_COMPONENTS, e); } } @GET - @Path(RestPaths.LIVE_COMPONENTS+"/{component}") + @Path(LIVE_COMPONENTS + "/{component}") @Produces({MediaType.APPLICATION_JSON}) public SerializedComponentInformation getLiveComponent( @PathParam("component") String component) { @@ -179,11 +247,16 @@ public class ApplicationResource extends AbstractSliderResource { } catch (YarnRuntimeException e) { throw new NotFoundException("Unknown component: " + component); } catch (Exception e) { - throw buildException(RestPaths.LIVE_CONTAINERS, e); + throw buildException(LIVE_CONTAINERS, e); } } - - List<RoleInstance> lookupRoleContainers(String component) { + + /** + * Look up all containers of a specific component name + * @param component component/role name + * @return list of instances. This is a snapshot + */ + private List<RoleInstance> lookupRoleContainers(String component) { RoleStatus roleStatus = state.lookupRoleStatus(component); List<RoleInstance> ownedContainerList = state.cloneOwnedContainerList(); List<RoleInstance> matching = new ArrayList<RoleInstance>(ownedContainerList.size()); @@ -195,5 +268,21 @@ public class ApplicationResource extends AbstractSliderResource { } return matching; } - + + protected AggregateConf lookupAggregateConf(String key) { + try { + return (AggregateConf) cache.lookup(key); + } catch (Exception e) { + throw buildException(key, e); + } + } + + protected ConfTree lookupConfTree(String key) { + try { + return (ConfTree) cache.lookup(key); + } catch (Exception e) { + throw buildException(key, e); + } + } + } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0d2a35bb/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/AggregateModelRefresher.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/AggregateModelRefresher.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/AggregateModelRefresher.java new file mode 100644 index 0000000..ee28abf --- /dev/null +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/AggregateModelRefresher.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.server.appmaster.web.rest.application.resources; + +import org.apache.slider.core.conf.AggregateConf; +import org.apache.slider.server.appmaster.state.StateAccessForProviders; + +/** + * Refresh the aggregate desired model via + * {@link StateAccessForProviders#getInstanceDefinitionSnapshot()} + */ +public class AggregateModelRefresher + implements ResourceRefresher<AggregateConf> { + + private final StateAccessForProviders state; + private final boolean resolved; + + public AggregateModelRefresher(StateAccessForProviders state, + boolean resolved) { + this.state = state; + this.resolved = resolved; + } + + @Override + public AggregateConf refresh() throws Exception { + return + resolved ? + state.getInstanceDefinitionSnapshot() + : state.getUnresolvedInstanceDefinition(); + } +} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0d2a35bb/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/AppconfRefresher.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/AppconfRefresher.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/AppconfRefresher.java new file mode 100644 index 0000000..65582ed --- /dev/null +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/AppconfRefresher.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.server.appmaster.web.rest.application.resources; + +import org.apache.slider.core.conf.AggregateConf; +import org.apache.slider.core.conf.ConfTree; +import org.apache.slider.core.persist.ConfTreeSerDeser; +import org.apache.slider.server.appmaster.state.StateAccessForProviders; + +/** + * refresher for resources and application configuration + */ +public class AppconfRefresher + implements ResourceRefresher<ConfTree> { + + private final StateAccessForProviders state; + private final boolean unresolved; + private final boolean resources; + + public AppconfRefresher(StateAccessForProviders state, + boolean unresolved, + boolean resources) { + this.state = state; + this.unresolved = unresolved; + this.resources = resources; + } + + + @Override + public ConfTree refresh() throws Exception { + AggregateConf aggregateConf = + unresolved ? + state.getUnresolvedInstanceDefinition(): + state.getInstanceDefinitionSnapshot(); + ConfTree ct = resources ? aggregateConf.getResources() + : aggregateConf.getResources(); + return new ConfTreeSerDeser().fromInstance(ct); + } +} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0d2a35bb/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/CachedContent.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/CachedContent.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/CachedContent.java index 78e65e8..22fd0fe 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/CachedContent.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/CachedContent.java @@ -49,7 +49,7 @@ public class CachedContent<T> { * Get the value, triggering a refresh if needed * @return the cached or latest value. */ - public T get() { + public T get() throws Exception { maybeRefresh(); return getCachedValue(); } @@ -78,7 +78,7 @@ public class CachedContent<T> { * Maybe refresh the content * @return true if a refresh took place. */ - public synchronized boolean maybeRefresh() { + public synchronized boolean maybeRefresh() throws Exception { long now = now(); if (cachedValue == null || now >= expires) { log.debug("Refreshing at time {}", now); @@ -97,7 +97,7 @@ public class CachedContent<T> { * Force a refresh and reset the expiry counter * @return the new value */ - protected synchronized T forceRefresh() { + protected synchronized T forceRefresh() throws Exception { refreshCounter ++; T updated = refresh.refresh(); Preconditions.checkNotNull(updated); http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0d2a35bb/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/ContentCache.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/ContentCache.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/ContentCache.java index 6ee9604..b60e9b3 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/ContentCache.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/ContentCache.java @@ -19,7 +19,6 @@ package org.apache.slider.server.appmaster.web.rest.application.resources; import com.google.common.base.Preconditions; -import org.apache.slider.server.appmaster.web.rest.RestPaths; import java.util.concurrent.ConcurrentHashMap; @@ -35,7 +34,7 @@ public class ContentCache extends ConcurrentHashMap<String, CachedContent> { public ContentCache() { } - public Object lookup(String key) { + public Object lookup(String key) throws Exception { CachedContent content = get(key); Preconditions.checkNotNull(content, "no content for path " + key); return content.get(); http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0d2a35bb/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/LiveComponentsRefresher.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/LiveComponentsRefresher.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/LiveComponentsRefresher.java index e543265..530bbd3 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/LiveComponentsRefresher.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/LiveComponentsRefresher.java @@ -35,7 +35,8 @@ public class LiveComponentsRefresher } @Override - public Map<String, SerializedComponentInformation> refresh() { + public Map<String, SerializedComponentInformation> refresh() throws + Exception { Map<Integer, RoleStatus> roleStatusMap = state.getRoleStatusMap(); Map<String, SerializedComponentInformation> results = http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0d2a35bb/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/LiveContainersRefresher.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/LiveContainersRefresher.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/LiveContainersRefresher.java index 39a543b..1b29801 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/LiveContainersRefresher.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/LiveContainersRefresher.java @@ -38,7 +38,8 @@ public class LiveContainersRefresher implements ResourceRefresher<Map<String, Se } @Override - public Map<String, SerializedContainerInformation> refresh() { + public Map<String, SerializedContainerInformation> refresh() throws + Exception { List<RoleInstance> containerList = state.cloneOwnedContainerList(); Map<String, SerializedContainerInformation> map = new HashMap<String, SerializedContainerInformation>(); http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0d2a35bb/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/LiveResourcesRefresher.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/LiveResourcesRefresher.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/LiveResourcesRefresher.java index dd845d8..f988297 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/LiveResourcesRefresher.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/LiveResourcesRefresher.java @@ -35,7 +35,7 @@ public class LiveResourcesRefresher implements ResourceRefresher<ConfTree> { } @Override - public ConfTree refresh() { + public ConfTree refresh() throws Exception { // snapshot resources ConfTreeOperations resources = state.getResourcesSnapshot(); http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0d2a35bb/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/ResourceRefresher.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/ResourceRefresher.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/ResourceRefresher.java index 9822fbc..35f0367 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/ResourceRefresher.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/ResourceRefresher.java @@ -27,5 +27,5 @@ public interface ResourceRefresher<T> { * Build an up to date version of the data * @return a new instance of the (JSON serializable) data */ - T refresh(); + T refresh() throws Exception; } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0d2a35bb/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/ResourceSnapshotRefresher.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/ResourceSnapshotRefresher.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/ResourceSnapshotRefresher.java index 0034d57..c16912a 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/ResourceSnapshotRefresher.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/ResourceSnapshotRefresher.java @@ -31,7 +31,7 @@ public class ResourceSnapshotRefresher implements ResourceRefresher<ConfTree> { } @Override - public ConfTree refresh() { + public ConfTree refresh() throws Exception { // snapshot resources ConfTreeOperations resources = state.getResourcesSnapshot(); http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0d2a35bb/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAgentWeb.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAgentWeb.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAgentWeb.groovy index 2ad7129..6df0452 100644 --- a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAgentWeb.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAgentWeb.groovy @@ -26,6 +26,9 @@ import org.apache.slider.agent.AgentMiniClusterTestBase import org.apache.slider.api.StateValues import org.apache.slider.api.types.SerializedComponentInformation import org.apache.slider.api.types.SerializedContainerInformation +import org.apache.slider.common.params.Arguments +import org.apache.slider.core.conf.AggregateConf +import org.apache.slider.core.conf.ConfTree import org.apache.slider.server.appmaster.web.rest.application.ApplicationResource import static org.apache.slider.api.ResourceKeys.* @@ -45,6 +48,8 @@ import static org.apache.slider.server.appmaster.management.MetricsKeys.* class TestStandaloneAgentWeb extends AgentMiniClusterTestBase { public static final int WEB_STARTUP_TIME = 30000 + public static final String TEST_GLOBAL_OPTION = "test.global.option" + public static final String TEST_GLOBAL_OPTION_PRESENT = "present" @Test public void testStandaloneAgentWeb() throws Throwable { @@ -58,7 +63,10 @@ class TestStandaloneAgentWeb extends AgentMiniClusterTestBase { ServiceLauncher<SliderClient> launcher = - createStandaloneAM(clustername, true, false) + createStandaloneAMWithArgs(clustername, + [Arguments.ARG_OPTION, + TEST_GLOBAL_OPTION, TEST_GLOBAL_OPTION_PRESENT], + true, false) SliderClient client = launcher.service addToTeardown(client); @@ -139,11 +147,8 @@ class TestStandaloneAgentWeb extends AgentMiniClusterTestBase { assert amContainerInfo.state == StateValues.STATE_LIVE describe "base entry lists" - def list = fetchType(ArrayList, appmaster, LIVE) - def live_entries = ApplicationResource.LIVE_ENTRIES - assert list.size() == live_entries.size() - live_entries.containsAll(list) + assertPathServesList(appmaster, LIVE, ApplicationResource.LIVE_ENTRIES) describe "containers" @@ -180,6 +185,55 @@ class TestStandaloneAgentWeb extends AgentMiniClusterTestBase { assert amFullInfo.containers.size() == 1 assert amFullInfo.containers[0] == amContainerId + testRESTModel(appmaster) + + + } + + public void testRESTModel(String appmaster) { + describe "model" + + assertPathServesList(appmaster, + MODEL, + ApplicationResource.MODEL_ENTRIES) + + def unresolvedConf = fetchType(AggregateConf, appmaster, MODEL_DESIRED) + log.info "Unresolved \n$unresolvedConf" + def unresolvedAppConf = unresolvedConf.appConfOperations + + def sam = "slider-appmaster" + assert unresolvedAppConf.getComponentOpt(sam, + TEST_GLOBAL_OPTION, "") == "" + def resolvedConf = fetchType(AggregateConf, appmaster, MODEL_RESOLVED) + log.info "Resolved \n$resolvedConf" + assert resolvedConf.appConfOperations.getComponentOpt( + sam, TEST_GLOBAL_OPTION, "") == TEST_GLOBAL_OPTION_PRESENT + + def unresolved = fetchTypeList(ConfTree, appmaster, + [MODEL_DESIRED_APPCONF, MODEL_DESIRED_RESOURCES]) + assert unresolved[0].components[sam][TEST_GLOBAL_OPTION] == null + + + def resolved = fetchTypeList(ConfTree, appmaster, + [MODEL_RESOLVED_APPCONF, MODEL_RESOLVED_RESOURCES]) + assert resolved[0].components[sam][TEST_GLOBAL_OPTION] == + TEST_GLOBAL_OPTION_PRESENT + } + + /** + * Assert that a path resolves to an array list that contains + * those entries (and only those entries) expected + * @param appmaster AM ref + * @param path path under AM + * @param entries entries to assert the presence of + */ + public void assertPathServesList( + String appmaster, + String path, + List<String> entries) { + def list = fetchType(ArrayList, appmaster, path) + assert list.size() == entries.size() + assert entries.containsAll(list) } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0d2a35bb/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAppRestIntegration.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAppRestIntegration.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAppRestIntegration.groovy index 712445d..cf32f33 100644 --- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAppRestIntegration.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAppRestIntegration.groovy @@ -128,7 +128,7 @@ class TestMockAppStateAppRestIntegration extends BaseMockAppStateTest implements class IntRefresher implements ResourceRefresher<Integer> { int count ; @Override - Integer refresh() { + Integer refresh() throws Exception { log.info("Refresh at $count") def result = count count += 1; http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0d2a35bb/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy b/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy index f9857a8..806285b 100644 --- a/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy @@ -1184,4 +1184,21 @@ class SliderTestUtils extends Assert { ConfTreeOperations tree = new ConfTreeOperations(ctree) return tree } + + /** + * Fetch a list of URLs, all of which must be of the same type + * @param clazz class of resolved values + * @param appmaster URL to app master + * @param subpaths list of subpaths + * @return a list of values in the same order as the paths passed in + */ + public <T> List<T> fetchTypeList( + Class<T> clazz, String appmaster, List<String> subpaths + ) { + List<T> results = [] + subpaths.each { String it -> + results.add(fetchType(clazz, appmaster, it)) + } + return results; + } }
