http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/84134031/slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderClusterProtocolProxy.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderClusterProtocolProxy.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderClusterProtocolProxy.java index 3f4c16b..39f1349 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderClusterProtocolProxy.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderClusterProtocolProxy.java @@ -201,5 +201,117 @@ public class SliderClusterProtocolProxy implements SliderClusterProtocol { return endpoint.getLivenessInformation(NULL_CONTROLLER, request); } catch (ServiceException e) { throw convert(e); - } } + } + } + + @Override + public Messages.GetLiveContainersResponseProto getLiveContainers(Messages.GetLiveContainersRequestProto request) throws + IOException { + try { + return endpoint.getLiveContainers(NULL_CONTROLLER, request); + } catch (ServiceException e) { + throw convert(e); + } + } + + @Override + public Messages.ContainerInformationProto getLiveContainer(Messages.GetLiveContainerRequestProto request) throws + IOException { + try { + return endpoint.getLiveContainer(NULL_CONTROLLER, request); + } catch (ServiceException e) { + throw convert(e); + } + } + + @Override + public Messages.GetLiveComponentsResponseProto getLiveComponents(Messages.GetLiveComponentsRequestProto request) throws + IOException { + try { + return endpoint.getLiveComponents(NULL_CONTROLLER, request); + } catch (ServiceException e) { + throw convert(e); + } + } + + @Override + public Messages.ComponentInformationProto getLiveComponent(Messages.GetLiveComponentRequestProto request) throws + IOException { + try { + return endpoint.getLiveComponent(NULL_CONTROLLER, request); + } catch (ServiceException e) { + throw convert(e); + } + } + + @Override + public Messages.WrappedJsonProto getModelDesired(RpcController controller, + Messages.EmptyPayloadProto request) throws IOException { + try { + return endpoint.getModelDesired(NULL_CONTROLLER, request); + } catch (ServiceException e) { + throw convert(e); + } + } + + @Override + public Messages.WrappedJsonProto getModelDesiredAppconf(RpcController controller, + Messages.EmptyPayloadProto request) throws IOException { + try { + return endpoint.getModelDesiredAppconf(NULL_CONTROLLER, request); + } catch (ServiceException e) { + throw convert(e); + } + } + + @Override + public Messages.WrappedJsonProto getModelDesiredResources(RpcController controller, + Messages.EmptyPayloadProto request) throws IOException { + try { + return endpoint.getModelDesiredResources(NULL_CONTROLLER, request); + } catch (ServiceException e) { + throw convert(e); + } + } + + @Override + public Messages.WrappedJsonProto getModelResolved(RpcController controller, + Messages.EmptyPayloadProto request) throws IOException { + try { + return endpoint.getModelResolved(NULL_CONTROLLER, request); + } catch (ServiceException e) { + throw convert(e); + } + } + + @Override + public Messages.WrappedJsonProto getModelResolvedAppconf(RpcController controller, + Messages.EmptyPayloadProto request) throws IOException { + try { + return endpoint.getModelResolvedAppconf(NULL_CONTROLLER, request); + } catch (ServiceException e) { + throw convert(e); + } + } + + @Override + public Messages.WrappedJsonProto getModelResolvedResources(RpcController controller, + Messages.EmptyPayloadProto request) throws IOException { + try { + return endpoint.getModelResolvedResources(NULL_CONTROLLER, request); + } catch (ServiceException e) { + throw convert(e); + } + } + + @Override + public Messages.WrappedJsonProto getLiveResources(RpcController controller, + Messages.EmptyPayloadProto request) throws IOException { + try { + return endpoint.getLiveResources(NULL_CONTROLLER, request); + } catch (ServiceException e) { + throw convert(e); + } + + } }
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/84134031/slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderIPCService.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderIPCService.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderIPCService.java index 596edaf..5a9f319 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderIPCService.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderIPCService.java @@ -19,19 +19,23 @@ package org.apache.slider.server.appmaster.rpc; import com.google.common.base.Preconditions; +import com.google.protobuf.RpcController; import org.apache.hadoop.ipc.ProtocolSignature; import org.apache.hadoop.service.AbstractService; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.slider.api.ClusterDescription; import org.apache.slider.api.SliderClusterProtocol; import org.apache.slider.api.proto.Messages; - import org.apache.slider.api.types.ApplicationLivenessInformation; +import org.apache.slider.api.types.ComponentInformation; +import org.apache.slider.api.types.ContainerInformation; import org.apache.slider.core.conf.AggregateConf; import org.apache.slider.core.conf.ConfTree; import org.apache.slider.core.exceptions.ServiceNotReadyException; import org.apache.slider.core.main.LauncherExitCodes; +import org.apache.slider.core.persist.AggregateConfSerDeser; import org.apache.slider.core.persist.ConfTreeSerDeser; import org.apache.slider.server.appmaster.AppMasterActionOperations; import org.apache.slider.server.appmaster.actions.ActionFlexCluster; @@ -43,18 +47,32 @@ import org.apache.slider.server.appmaster.actions.QueueAccess; import org.apache.slider.server.appmaster.management.MetricsAndMonitoring; import org.apache.slider.server.appmaster.state.RoleInstance; import org.apache.slider.server.appmaster.state.StateAccessForProviders; +import org.apache.slider.server.appmaster.web.rest.application.resources.ContentCache; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.FileNotFoundException; import java.io.IOException; import java.util.List; +import java.util.Map; import java.util.concurrent.TimeUnit; -import static org.apache.slider.api.proto.RestTypeMarshalling.*; +import static org.apache.slider.api.proto.RestTypeMarshalling.marshall; +import static org.apache.slider.server.appmaster.web.rest.RestPaths.LIVE_COMPONENTS; +import static org.apache.slider.server.appmaster.web.rest.RestPaths.LIVE_CONTAINERS; +import static org.apache.slider.server.appmaster.web.rest.RestPaths.LIVE_RESOURCES; +import static org.apache.slider.server.appmaster.web.rest.RestPaths.MODEL_DESIRED; +import static org.apache.slider.server.appmaster.web.rest.RestPaths.MODEL_DESIRED_APPCONF; +import static org.apache.slider.server.appmaster.web.rest.RestPaths.MODEL_DESIRED_RESOURCES; +import static org.apache.slider.server.appmaster.web.rest.RestPaths.MODEL_RESOLVED; +import static org.apache.slider.server.appmaster.web.rest.RestPaths.MODEL_RESOLVED_APPCONF; +import static org.apache.slider.server.appmaster.web.rest.RestPaths.MODEL_RESOLVED_RESOURCES; /** * Implement the {@link SliderClusterProtocol}. */ +@SuppressWarnings("unchecked") + public class SliderIPCService extends AbstractService implements SliderClusterProtocol { @@ -65,7 +83,8 @@ public class SliderIPCService extends AbstractService private final StateAccessForProviders state; private final MetricsAndMonitoring metricsAndMonitoring; private final AppMasterActionOperations amOperations; - + private final ContentCache cache; + /** * This is the prefix used for metrics */ @@ -78,20 +97,24 @@ public class SliderIPCService extends AbstractService * @param state state view * @param actionQueues queues for actions * @param metricsAndMonitoring metrics + * @param cache */ public SliderIPCService(AppMasterActionOperations amOperations, StateAccessForProviders state, QueueAccess actionQueues, - MetricsAndMonitoring metricsAndMonitoring) { + MetricsAndMonitoring metricsAndMonitoring, ContentCache cache) { super("SliderIPCService"); Preconditions.checkArgument(amOperations != null, "null amOperations"); Preconditions.checkArgument(state != null, "null appState"); Preconditions.checkArgument(actionQueues != null, "null actionQueues"); - Preconditions.checkArgument(metricsAndMonitoring != null, "null metricsAndMonitoring"); + Preconditions.checkArgument(metricsAndMonitoring != null, + "null metricsAndMonitoring"); + Preconditions.checkArgument(cache != null, "null cache"); this.state = state; this.actionQueues = actionQueues; this.metricsAndMonitoring = metricsAndMonitoring; this.amOperations = amOperations; + this.cache = cache; } @Override //SliderClusterProtocol @@ -312,4 +335,139 @@ public class SliderIPCService extends AbstractService state.getApplicationLivenessInformation(); return marshall(info); } + + @Override + public Messages.GetLiveContainersResponseProto getLiveContainers( + Messages.GetLiveContainersRequestProto request) + throws IOException { + Map<String, ContainerInformation> infoMap = + (Map<String, ContainerInformation>) cache.lookupWithIOE( + LIVE_CONTAINERS); + Messages.GetLiveContainersResponseProto.Builder builder = + Messages.GetLiveContainersResponseProto.newBuilder(); + + for (Map.Entry<String, ContainerInformation> entry : infoMap + .entrySet()) { + builder.addNames(entry.getKey()); + builder.addContainers(marshall(entry.getValue())); + } + return builder.build(); + } + + @Override + public Messages.ContainerInformationProto getLiveContainer(Messages.GetLiveContainerRequestProto request) throws + IOException { + String containerId = request.getContainerId(); + RoleInstance id = state.getLiveInstanceByContainerID(containerId); + ContainerInformation containerInformation = id.serialize(); + return marshall(containerInformation); + } + + @Override + public Messages.GetLiveComponentsResponseProto getLiveComponents(Messages.GetLiveComponentsRequestProto request) throws + IOException { + Map<String, ComponentInformation> infoMap = + (Map<String, ComponentInformation>) cache.lookupWithIOE( + LIVE_COMPONENTS); + Messages.GetLiveComponentsResponseProto.Builder builder = + Messages.GetLiveComponentsResponseProto.newBuilder(); + + for (Map.Entry<String, ComponentInformation> entry : infoMap + .entrySet()) { + builder.addNames(entry.getKey()); + builder.addComponents(marshall(entry.getValue())); + } + return builder.build(); + } + + @Override + public Messages.WrappedJsonProto getModelDesired(RpcController controller, + Messages.EmptyPayloadProto request) throws IOException { + return lookupAggregateConf(MODEL_DESIRED); + } + + @Override + public Messages.WrappedJsonProto getModelDesiredAppconf(RpcController controller, + Messages.EmptyPayloadProto request) throws IOException { + return lookupConfTree(MODEL_DESIRED_APPCONF); + } + + @Override + public Messages.WrappedJsonProto getModelDesiredResources(RpcController controller, + Messages.EmptyPayloadProto request) throws IOException { + return lookupConfTree(MODEL_DESIRED_RESOURCES); + } + + @Override + public Messages.WrappedJsonProto getModelResolved(RpcController controller, + Messages.EmptyPayloadProto request) throws IOException { + return lookupAggregateConf(MODEL_RESOLVED); + } + + @Override + public Messages.WrappedJsonProto getModelResolvedAppconf(RpcController controller, + Messages.EmptyPayloadProto request) throws IOException { + return lookupConfTree(MODEL_RESOLVED_APPCONF); + } + + @Override + public Messages.WrappedJsonProto getModelResolvedResources(RpcController controller, + Messages.EmptyPayloadProto request) throws IOException { + return lookupConfTree(MODEL_RESOLVED_RESOURCES); + } + + @Override + public Messages.WrappedJsonProto getLiveResources(RpcController controller, + Messages.EmptyPayloadProto request) throws IOException { + return lookupConfTree(LIVE_RESOURCES); + } + + @Override + public Messages.ComponentInformationProto getLiveComponent(Messages.GetLiveComponentRequestProto request) throws + IOException { + String name = request.getName(); + try { + return marshall(state.getComponentInformation(name)); + } catch (YarnRuntimeException e) { + throw new FileNotFoundException("Unknown component: " + name); + } + } + + /** + * Helper method; look up an aggregate configuration in the cache from + * a key, or raise an exception + * @param key key to resolve + * @return the configuration + * @throws IOException on a failure + */ + + protected Messages.WrappedJsonProto lookupAggregateConf(String key) throws + IOException { + AggregateConf aggregateConf = (AggregateConf) cache.lookupWithIOE(key); + String json = AggregateConfSerDeser.toString(aggregateConf); + return wrap(json); + } + + /** + * Helper method; look up an conf tree in the cache from + * a key, or raise an exception + * @param key key to resolve + * @return the configuration + * @throws IOException on a failure + */ + protected Messages.WrappedJsonProto lookupConfTree(String key) throws + IOException { + ConfTree conf = (ConfTree) cache.lookupWithIOE(key); + String json = ConfTreeSerDeser.toString(conf); + return wrap(json); + } + + private Messages.WrappedJsonProto wrap(String json) { + Messages.WrappedJsonProto.Builder builder = + Messages.WrappedJsonProto.newBuilder(); + builder.setJson(json); + return builder.build(); + } + + } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/84134031/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ProviderAppState.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ProviderAppState.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ProviderAppState.java index b5b2cb7..d237f42 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ProviderAppState.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ProviderAppState.java @@ -257,4 +257,32 @@ public class ProviderAppState implements StateAccessForProviders { return nodes; } + @Override + public List<RoleInstance> lookupRoleContainers(String component) { + RoleStatus roleStatus = lookupRoleStatus(component); + List<RoleInstance> ownedContainerList = cloneOwnedContainerList(); + List<RoleInstance> matching = + new ArrayList<RoleInstance>(ownedContainerList.size()); + int roleId = roleStatus.getPriority(); + for (RoleInstance instance : ownedContainerList) { + if (instance.roleId == roleId) { + matching.add(instance); + } + } + return matching; + } + + @Override + public ComponentInformation getComponentInformation(String component) { + RoleStatus roleStatus = lookupRoleStatus(component); + ComponentInformation info = roleStatus.serialize(); + List<RoleInstance> containers = lookupRoleContainers(component); + info.containers = new ArrayList<String>(containers.size()); + for (RoleInstance container : containers) { + info.containers.add(container.id); + } + return info; + + } + } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/84134031/slider-core/src/main/java/org/apache/slider/server/appmaster/state/StateAccessForProviders.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/StateAccessForProviders.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/StateAccessForProviders.java index 0ea5428..eef178a 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/StateAccessForProviders.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/StateAccessForProviders.java @@ -272,4 +272,13 @@ public interface StateAccessForProviders { * @return a list of nodes, may be empty */ List<RoleInstance> enumLiveNodesInRole(String role); + + /** + * Look up all containers of a specific component name + * @param component component/role name + * @return list of instances. This is a snapshot + */ + List<RoleInstance> lookupRoleContainers(String component); + + ComponentInformation getComponentInformation(String component); } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/84134031/slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApi.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApi.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApi.java index f1bff64..179ae8c 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApi.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApi.java @@ -26,6 +26,7 @@ import org.apache.slider.server.appmaster.state.AppState; import org.apache.slider.server.appmaster.state.RoleStatus; import org.apache.slider.server.appmaster.state.StateAccessForProviders; import org.apache.slider.server.appmaster.web.rest.agent.AgentRestOperations; +import org.apache.slider.server.appmaster.web.rest.application.resources.ContentCache; import org.apache.slider.server.services.security.CertificateManager; import java.util.Map; @@ -81,4 +82,6 @@ public interface WebAppApi { QueueAccess getQueues(); AppMasterActionOperations getAMOperations(); + + ContentCache getContentCache(); } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/84134031/slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApiImpl.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApiImpl.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApiImpl.java index a7d85af..0730a21 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApiImpl.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApiImpl.java @@ -24,6 +24,7 @@ import org.apache.slider.server.appmaster.management.MetricsAndMonitoring; import org.apache.slider.server.appmaster.state.RoleStatus; import org.apache.slider.server.appmaster.state.StateAccessForProviders; import org.apache.slider.server.appmaster.web.rest.agent.AgentRestOperations; +import org.apache.slider.server.appmaster.web.rest.application.resources.ContentCache; import org.apache.slider.server.services.security.CertificateManager; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -47,6 +48,7 @@ public class WebAppApiImpl implements WebAppApi { private final MetricsAndMonitoring metricsAndMonitoring; private final QueueAccess queues; private final AppMasterActionOperations appMasterOperations; + private final ContentCache contentCache; public WebAppApiImpl(StateAccessForProviders appState, ProviderService provider, @@ -54,8 +56,10 @@ public class WebAppApiImpl implements WebAppApi { RegistryOperations registryOperations, MetricsAndMonitoring metricsAndMonitoring, QueueAccess queues, - AppMasterActionOperations appMasterOperations) { + AppMasterActionOperations appMasterOperations, + ContentCache contentCache) { this.appMasterOperations = appMasterOperations; + this.contentCache = contentCache; checkNotNull(appState); checkNotNull(provider); this.queues = queues; @@ -117,4 +121,9 @@ public class WebAppApiImpl implements WebAppApi { public AppMasterActionOperations getAMOperations() { return appMasterOperations; } + + @Override + public ContentCache getContentCache() { + return contentCache; + } } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/84134031/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/ApplicationResouceContentCacheFactory.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/ApplicationResouceContentCacheFactory.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/ApplicationResouceContentCacheFactory.java new file mode 100644 index 0000000..e7b8fc7 --- /dev/null +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/ApplicationResouceContentCacheFactory.java @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.server.appmaster.web.rest.application; + +import org.apache.slider.api.types.ComponentInformation; +import org.apache.slider.api.types.ContainerInformation; +import org.apache.slider.core.conf.AggregateConf; +import org.apache.slider.core.conf.ConfTree; +import org.apache.slider.server.appmaster.state.StateAccessForProviders; +import org.apache.slider.server.appmaster.web.rest.application.resources.AggregateModelRefresher; +import org.apache.slider.server.appmaster.web.rest.application.resources.AppconfRefresher; +import org.apache.slider.server.appmaster.web.rest.application.resources.CachedContent; +import org.apache.slider.server.appmaster.web.rest.application.resources.ContentCache; +import org.apache.slider.server.appmaster.web.rest.application.resources.LiveComponentsRefresher; +import org.apache.slider.server.appmaster.web.rest.application.resources.LiveContainersRefresher; +import org.apache.slider.server.appmaster.web.rest.application.resources.LiveResourcesRefresher; +import org.apache.slider.server.appmaster.web.rest.application.resources.LiveStatisticsRefresher; + +import java.util.Map; + +import static org.apache.slider.server.appmaster.web.rest.RestPaths.LIVE_COMPONENTS; +import static org.apache.slider.server.appmaster.web.rest.RestPaths.LIVE_CONTAINERS; +import static org.apache.slider.server.appmaster.web.rest.RestPaths.LIVE_RESOURCES; +import static org.apache.slider.server.appmaster.web.rest.RestPaths.LIVE_STATISTICS; +import static org.apache.slider.server.appmaster.web.rest.RestPaths.MODEL_DESIRED; +import static org.apache.slider.server.appmaster.web.rest.RestPaths.MODEL_DESIRED_APPCONF; +import static org.apache.slider.server.appmaster.web.rest.RestPaths.MODEL_DESIRED_RESOURCES; +import static org.apache.slider.server.appmaster.web.rest.RestPaths.MODEL_RESOLVED; +import static org.apache.slider.server.appmaster.web.rest.RestPaths.MODEL_RESOLVED_APPCONF; +import static org.apache.slider.server.appmaster.web.rest.RestPaths.MODEL_RESOLVED_RESOURCES; + +public class ApplicationResouceContentCacheFactory { + public static final int LIFESPAN = 500; + + /** + * Build the content cache + * @param cache cache to construct + * @param state state view + */ + public static ContentCache createContentCache( + StateAccessForProviders state) { + ContentCache cache = new ContentCache(); + cache.put(LIVE_RESOURCES, + new CachedContent<ConfTree>(LIFESPAN, + new LiveResourcesRefresher(state))); + cache.put(LIVE_CONTAINERS, + new CachedContent<Map<String, ContainerInformation>>(LIFESPAN, + new LiveContainersRefresher(state))); + cache.put(LIVE_COMPONENTS, + new CachedContent<Map<String, ComponentInformation>>(LIFESPAN, + new LiveComponentsRefresher(state))); + cache.put(MODEL_DESIRED, + new CachedContent<AggregateConf>(LIFESPAN, + new AggregateModelRefresher(state, false))); + cache.put(MODEL_RESOLVED, + new CachedContent<AggregateConf>(LIFESPAN, + new AggregateModelRefresher(state, true))); + cache.put(MODEL_RESOLVED_APPCONF, + new CachedContent<ConfTree>(LIFESPAN, + new AppconfRefresher(state, false, false))); + cache.put(MODEL_RESOLVED_RESOURCES, + new CachedContent<ConfTree>(LIFESPAN, + new AppconfRefresher(state, false, true))); + cache.put(MODEL_DESIRED_APPCONF, + new CachedContent<ConfTree>(LIFESPAN, + new AppconfRefresher(state, true, false))); + cache.put(MODEL_DESIRED_RESOURCES, + new CachedContent<ConfTree>(LIFESPAN, + new AppconfRefresher(state, true, true))); + cache.put(LIVE_STATISTICS, + new CachedContent<Map<String, Integer>>(LIFESPAN, + new LiveStatisticsRefresher(state))); + return cache; + } +} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/84134031/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/ApplicationResource.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/ApplicationResource.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/ApplicationResource.java index 9fd59c1..2dedfb8 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/ApplicationResource.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/ApplicationResource.java @@ -28,7 +28,6 @@ import org.apache.slider.core.conf.AggregateConf; import org.apache.slider.core.conf.ConfTree; import org.apache.slider.core.exceptions.NoSuchNodeException; import org.apache.slider.server.appmaster.state.RoleInstance; -import org.apache.slider.server.appmaster.state.RoleStatus; import org.apache.slider.server.appmaster.state.StateAccessForProviders; import org.apache.slider.server.appmaster.web.WebAppApi; import org.apache.slider.server.appmaster.web.rest.AbstractSliderResource; @@ -36,16 +35,9 @@ import static org.apache.slider.server.appmaster.web.rest.RestPaths.*; import org.apache.slider.server.appmaster.web.rest.application.actions.RestActionStop; import org.apache.slider.server.appmaster.web.rest.application.actions.StopResponse; -import org.apache.slider.server.appmaster.web.rest.application.resources.AggregateModelRefresher; -import org.apache.slider.server.appmaster.web.rest.application.resources.AppconfRefresher; -import org.apache.slider.server.appmaster.web.rest.application.resources.CachedContent; -import org.apache.slider.server.appmaster.web.rest.application.resources.LiveContainersRefresher; import org.apache.slider.server.appmaster.web.rest.application.resources.ContentCache; -import org.apache.slider.server.appmaster.web.rest.application.resources.LiveComponentsRefresher; -import org.apache.slider.server.appmaster.web.rest.application.resources.LiveResourcesRefresher; import org.apache.slider.server.appmaster.web.rest.application.actions.RestActionPing; import org.apache.slider.api.types.PingInformation; -import org.apache.slider.server.appmaster.web.rest.application.resources.LiveStatisticsRefresher; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -75,7 +67,6 @@ public class ApplicationResource extends AbstractSliderResource { private static final Logger log = LoggerFactory.getLogger(ApplicationResource.class); - public static final int LIFESPAN = 500; public static final List<String> LIVE_ENTRIES = toJsonList("resources", "containers", "components", @@ -95,42 +86,13 @@ public class ApplicationResource extends AbstractSliderResource { * so is never very out of date, yet many GETs don't * overload the rest of the system. */ - private final ContentCache cache = new ContentCache(); + private final ContentCache cache; private final StateAccessForProviders state; public ApplicationResource(WebAppApi slider) { super(slider); state = slider.getAppState(); - cache.put(LIVE_RESOURCES, - new CachedContent<ConfTree>(LIFESPAN, - new LiveResourcesRefresher(state))); - cache.put(LIVE_CONTAINERS, - new CachedContent<Map<String, ContainerInformation>>(LIFESPAN, - new LiveContainersRefresher(state))); - cache.put(LIVE_COMPONENTS, - new CachedContent<Map<String, ComponentInformation>> (LIFESPAN, - new LiveComponentsRefresher(state))); - cache.put(MODEL_DESIRED, - new CachedContent<AggregateConf>(LIFESPAN, - new AggregateModelRefresher(state, false))); - cache.put(MODEL_RESOLVED, - new CachedContent<AggregateConf>(LIFESPAN, - new AggregateModelRefresher(state, true))); - cache.put(MODEL_RESOLVED_APPCONF, - new CachedContent<ConfTree>(LIFESPAN, - new AppconfRefresher(state, false, false))); - cache.put(MODEL_RESOLVED_RESOURCES, - new CachedContent<ConfTree>(LIFESPAN, - new AppconfRefresher(state, false, true))); - cache.put(MODEL_DESIRED_APPCONF, - new CachedContent<ConfTree>(LIFESPAN, - new AppconfRefresher(state, true, false))); - cache.put(MODEL_DESIRED_RESOURCES, - new CachedContent<ConfTree>(LIFESPAN, - new AppconfRefresher(state, true, true))); - cache.put(LIVE_STATISTICS, - new CachedContent<Map<String, Integer>>(LIFESPAN, - new LiveStatisticsRefresher(state))); + cache = slider.getContentCache(); } /** @@ -275,14 +237,7 @@ public class ApplicationResource extends AbstractSliderResource { @PathParam("component") String component) { markGet(SLIDER_SUBPATH_APPLICATION, LIVE_COMPONENTS); try { - RoleStatus roleStatus = state.lookupRoleStatus(component); - ComponentInformation info = roleStatus.serialize(); - List<RoleInstance> containers = lookupRoleContainers(component); - info.containers = new ArrayList<String>(containers.size()); - for (RoleInstance container : containers) { - info.containers.add(container.id); - } - return info; + return state.getComponentInformation(component); } catch (YarnRuntimeException e) { throw new NotFoundException("Unknown component: " + component); } catch (Exception e) { @@ -344,24 +299,6 @@ TODO: decide what structure to return here, then implement throw buildException(LIVE_STATISTICS, e); } } - - /** - * Look up all containers of a specific component name - * @param component component/role name - * @return list of instances. This is a snapshot - */ - private List<RoleInstance> lookupRoleContainers(String component) { - RoleStatus roleStatus = state.lookupRoleStatus(component); - List<RoleInstance> ownedContainerList = state.cloneOwnedContainerList(); - List<RoleInstance> matching = new ArrayList<RoleInstance>(ownedContainerList.size()); - int roleId = roleStatus.getPriority(); - for (RoleInstance instance : ownedContainerList) { - if (instance.roleId == roleId) { - matching.add(instance); - } - } - return matching; - } /** * Helper method; look up an aggregate configuration in the cache from @@ -378,6 +315,14 @@ TODO: decide what structure to return here, then implement } } + + /** + * Helper method; look up an conf tree in the cache from + * a key, or raise an exception + * @param key key to resolve + * @return the configuration + * @throws WebApplicationException on a failure + */ protected ConfTree lookupConfTree(String key) { try { return (ConfTree) cache.lookup(key); http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/84134031/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/ContentCache.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/ContentCache.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/ContentCache.java index b60e9b3..8f026a1 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/ContentCache.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/ContentCache.java @@ -18,8 +18,8 @@ package org.apache.slider.server.appmaster.web.rest.application.resources; -import com.google.common.base.Preconditions; - +import java.io.FileNotFoundException; +import java.io.IOException; import java.util.concurrent.ConcurrentHashMap; /** @@ -33,10 +33,35 @@ public class ContentCache extends ConcurrentHashMap<String, CachedContent> { public ContentCache() { } - + + public Object lookup(String key) throws Exception { CachedContent content = get(key); - Preconditions.checkNotNull(content, "no content for path " + key); + if (content == null) { + throw new FileNotFoundException("no content for path " + key); + } return content.get(); } + + + /** + * Lookup a cached item. If an exception is raised on the refresh... + * <ol> + * <li>IOExceptions are thrown directly</li> + * <li>Other exceptions are wrapped with an IOExceptions</li> + * </ol> + * @param key + * @return + * @throws IOException + */ + public Object lookupWithIOE(String key) throws IOException { + try { + return lookup(key); + } catch (IOException e) { + throw e; + } catch (Exception e) { + throw new IOException("Looking up " + key + ": " + e, e); + } + } + } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/84134031/slider-core/src/main/proto/SliderClusterMessages.proto ---------------------------------------------------------------------- diff --git a/slider-core/src/main/proto/SliderClusterMessages.proto b/slider-core/src/main/proto/SliderClusterMessages.proto index 2c31c2e..bcf3fb7 100644 --- a/slider-core/src/main/proto/SliderClusterMessages.proto +++ b/slider-core/src/main/proto/SliderClusterMessages.proto @@ -308,3 +308,13 @@ message GetLiveComponentRequestProto { message GetApplicationLivenessRequestProto { } + +message EmptyPayloadProto { +} + +/** + Generic JSON, often containing data structures serialized as a string +*/ +message WrappedJsonProto { + required string json = 1; +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/84134031/slider-core/src/main/proto/SliderClusterProtocol.proto ---------------------------------------------------------------------- diff --git a/slider-core/src/main/proto/SliderClusterProtocol.proto b/slider-core/src/main/proto/SliderClusterProtocol.proto index da09f3c..437e06f 100644 --- a/slider-core/src/main/proto/SliderClusterProtocol.proto +++ b/slider-core/src/main/proto/SliderClusterProtocol.proto @@ -136,5 +136,34 @@ service SliderClusterProtocolPB { rpc getLiveComponent(GetLiveComponentRequestProto) returns(ComponentInformationProto); +// AggregateConf getModelDesired() + rpc getModelDesired(EmptyPayloadProto) + returns(WrappedJsonProto); + + // ConfTree getModelDesiredAppconf + rpc getModelDesiredAppconf(EmptyPayloadProto) + returns(WrappedJsonProto); + + // ConfTree getModelDesiredResources + rpc getModelDesiredResources(EmptyPayloadProto) + returns(WrappedJsonProto); + +// AggregateConf getModelResolved() + rpc getModelResolved(EmptyPayloadProto) + returns(WrappedJsonProto); + + // ConfTree getModelResolvedAppconf + rpc getModelResolvedAppconf(EmptyPayloadProto) + returns(WrappedJsonProto); + + // ConfTree getModelResolvedResources + rpc getModelResolvedResources(EmptyPayloadProto) + returns(WrappedJsonProto); + + // ConfTree getLiveResources + rpc getLiveResources(EmptyPayloadProto) + returns(WrappedJsonProto); + + } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/84134031/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAppRestIntegration.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAppRestIntegration.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAppRestIntegration.groovy index e42f13c..5d9d1d1 100644 --- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAppRestIntegration.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAppRestIntegration.groovy @@ -117,7 +117,7 @@ class TestMockAppStateAppRestIntegration extends BaseMockAppStateTest implements WebAppApi api = new WebAppApiImpl(stateAccess, new MockProviderService(), null, null, - new MetricsAndMonitoring("metrics"), null, null) + new MetricsAndMonitoring("metrics"), null, null, null) return api } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/84134031/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/view/TestClusterSpecificationBlock.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/view/TestClusterSpecificationBlock.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/view/TestClusterSpecificationBlock.groovy index 62ceca5..9907b94 100644 --- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/view/TestClusterSpecificationBlock.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/view/TestClusterSpecificationBlock.groovy @@ -53,7 +53,7 @@ public class TestClusterSpecificationBlock { providerAppState, providerService, null, - null, null, null, null); + null, null, null, null, null); Injector injector = Guice.createInjector(new AbstractModule() { @Override http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/84134031/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/view/TestContainerStatsBlock.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/view/TestContainerStatsBlock.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/view/TestContainerStatsBlock.groovy index 16bf23b..4ef70a3 100644 --- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/view/TestContainerStatsBlock.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/view/TestContainerStatsBlock.groovy @@ -29,7 +29,6 @@ import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TABLE import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TR import org.apache.hadoop.yarn.webapp.hamlet.HamletImpl.EImp import org.apache.slider.api.ClusterNode -import org.apache.slider.api.SliderClusterProtocol import org.apache.slider.providers.ProviderService import org.apache.slider.server.appmaster.model.mock.* import org.apache.slider.server.appmaster.state.ProviderAppState @@ -62,7 +61,7 @@ public class TestContainerStatsBlock extends BaseMockAppStateTest { providerAppState, providerService, null, - null, metrics, null, null); + null, metrics, null, null, null); Injector injector = Guice.createInjector(new AbstractModule() { @Override http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/84134031/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/view/TestIndexBlock.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/view/TestIndexBlock.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/view/TestIndexBlock.groovy index eb68c8a..c44baee 100644 --- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/view/TestIndexBlock.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/view/TestIndexBlock.groovy @@ -24,7 +24,6 @@ import groovy.util.logging.Slf4j import org.apache.hadoop.yarn.api.records.Container import org.apache.hadoop.yarn.api.records.Priority import org.apache.hadoop.yarn.webapp.hamlet.Hamlet -import org.apache.slider.api.SliderClusterProtocol import org.apache.slider.providers.ProviderService import org.apache.slider.server.appmaster.model.mock.* import org.apache.slider.server.appmaster.state.ProviderAppState @@ -54,7 +53,7 @@ public class TestIndexBlock extends BaseMockAppStateTest { providerAppState, providerService, null, - null, metrics, null, null); + null, metrics, null, null, null); Injector injector = Guice.createInjector(new AbstractModule() { @Override http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/84134031/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/agent/TestAMAgentWebServices.java ---------------------------------------------------------------------- diff --git a/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/agent/TestAMAgentWebServices.java b/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/agent/TestAMAgentWebServices.java index edfebbd..d37fcea 100644 --- a/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/agent/TestAMAgentWebServices.java +++ b/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/agent/TestAMAgentWebServices.java @@ -144,7 +144,7 @@ public class TestAMAgentWebServices { slider = new WebAppApiImpl(providerAppState, new MockProviderService(), null, null, null, - null, null); + null, null, null); MapOperations compOperations = new MapOperations(); http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/84134031/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/management/TestAMManagementWebServices.java ---------------------------------------------------------------------- diff --git a/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/management/TestAMManagementWebServices.java b/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/management/TestAMManagementWebServices.java index badfbc3..90d40c6 100644 --- a/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/management/TestAMManagementWebServices.java +++ b/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/management/TestAMManagementWebServices.java @@ -192,7 +192,7 @@ public class TestAMManagementWebServices extends JerseyTest { slider = new WebAppApiImpl(providerAppState, new MockProviderService(), null, null, null, - null, null); + null, null, null); bind(SliderJacksonJaxbJsonProvider.class); bind(MockSliderAMWebServices.class);
