http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3aeab9ca/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java index 0f77824..eadb1dc 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java @@ -85,23 +85,9 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.atomic.AtomicInteger; -import static org.apache.slider.api.ResourceKeys.DEF_YARN_CORES; -import static org.apache.slider.api.ResourceKeys.DEF_YARN_LABEL_EXPRESSION; -import static org.apache.slider.api.ResourceKeys.DEF_YARN_MEMORY; -import static org.apache.slider.api.ResourceKeys.YARN_CORES; -import static org.apache.slider.api.ResourceKeys.YARN_LABEL_EXPRESSION; -import static org.apache.slider.api.ResourceKeys.YARN_MEMORY; -import static org.apache.slider.api.RoleKeys.ROLE_FAILED_INSTANCES; -import static org.apache.slider.api.RoleKeys.ROLE_FAILED_RECENTLY_INSTANCES; -import static org.apache.slider.api.RoleKeys.ROLE_FAILED_STARTING_INSTANCES; -import static org.apache.slider.api.RoleKeys.ROLE_NODE_FAILED_INSTANCES; -import static org.apache.slider.api.RoleKeys.ROLE_PREEMPTED_INSTANCES; -import static org.apache.slider.api.RoleKeys.ROLE_RELEASING_INSTANCES; -import static org.apache.slider.api.RoleKeys.ROLE_REQUESTED_INSTANCES; -import static org.apache.slider.api.StateValues.STATE_CREATED; -import static org.apache.slider.api.StateValues.STATE_DESTROYED; -import static org.apache.slider.api.StateValues.STATE_LIVE; -import static org.apache.slider.api.StateValues.STATE_SUBMITTED; +import static org.apache.slider.api.ResourceKeys.*; +import static org.apache.slider.api.RoleKeys.*; +import static org.apache.slider.api.StateValues.*; /** @@ -214,7 +200,6 @@ public class AppState { /** * Count of failed containers - */ private final Counter failedContainerCount = new Counter(); @@ -244,21 +229,21 @@ public class AppState { * resources, etc. When container started callback is received, * the node is promoted from here to the containerMap */ - private final Map<ContainerId, RoleInstance> startingNodes = + private final Map<ContainerId, RoleInstance> startingContainers = new ConcurrentHashMap<>(); /** * List of completed nodes. This isn't kept in the CD as it gets too * big for the RPC responses. Indeed, we should think about how deep to get this */ - private final Map<ContainerId, RoleInstance> completedNodes + private final Map<ContainerId, RoleInstance> completedContainers = new ConcurrentHashMap<>(); /** * Nodes that failed to start. * Again, kept out of the CD */ - private final Map<ContainerId, RoleInstance> failedNodes = + private final Map<ContainerId, RoleInstance> failedContainers = new ConcurrentHashMap<>(); /** @@ -324,8 +309,7 @@ public class AppState { private void register(String name, Counter counter) { this.metricsAndMonitoring.getMetrics().register( - MetricRegistry.name(AppState.class, - name), counter); + MetricRegistry.name(AppState.class, name), counter); } public long getFailedCountainerCount() { @@ -382,21 +366,20 @@ public class AppState { return rolePriorityMap; } - private Map<ContainerId, RoleInstance> getStartingNodes() { - return startingNodes; + private Map<ContainerId, RoleInstance> getStartingContainers() { + return startingContainers; } - private Map<ContainerId, RoleInstance> getCompletedNodes() { - return completedNodes; + private Map<ContainerId, RoleInstance> getCompletedContainers() { + return completedContainers; } - public Map<ContainerId, RoleInstance> getFailedNodes() { - return failedNodes; + public Map<ContainerId, RoleInstance> getFailedContainers() { + return failedContainers; } - - public Map<ContainerId, RoleInstance> getLiveNodes() { + public Map<ContainerId, RoleInstance> getLiveContainers() { return liveNodes; } @@ -592,8 +575,9 @@ public class AppState { initClusterStatus(); - // add the roles + // set up the role history roleHistory = new RoleHistory(providerRoles); + roleHistory.register(metricsAndMonitoring); roleHistory.onStart(fs, historyDir); //rebuild any live containers @@ -871,11 +855,11 @@ public class AppState { RoleInstance am = new RoleInstance(container); am.role = SliderKeys.COMPONENT_AM; am.roleId = SliderKeys.ROLE_AM_PRIORITY_INDEX; - am.createTime = System.currentTimeMillis(); - am.startTime = System.currentTimeMillis(); + am.createTime =now(); + am.startTime = am.createTime; appMasterNode = am; //it is also added to the set of live nodes - getLiveNodes().put(containerId, am); + getLiveContainers().put(containerId, am); putOwnedContainer(containerId, am); // patch up the role status @@ -894,7 +878,7 @@ public class AppState { * and it will just cause confusion */ public void noteAMLaunched() { - getLiveNodes().put(appMasterNode.getContainerId(), appMasterNode); + getLiveContainers().put(appMasterNode.getContainerId(), appMasterNode); } /** @@ -1002,7 +986,7 @@ public class AppState { */ public synchronized List<RoleInstance> cloneLiveContainerInfoList() { List<RoleInstance> allRoleInstances; - Collection<RoleInstance> values = getLiveNodes().values(); + Collection<RoleInstance> values = getLiveContainers().values(); allRoleInstances = new ArrayList<>(values); return allRoleInstances; } @@ -1015,7 +999,7 @@ public class AppState { */ public synchronized RoleInstance getLiveInstanceByContainerID(String containerId) throws NoSuchNodeException { - Collection<RoleInstance> nodes = getLiveNodes().values(); + Collection<RoleInstance> nodes = getLiveContainers().values(); return findNodeInCollection(containerId, nodes); } @@ -1062,7 +1046,7 @@ public class AppState { //first, a hashmap of those containerIDs is built up Set<String> uuidSet = new HashSet<String>(containerIDs); List<RoleInstance> nodes = new ArrayList<RoleInstance>(uuidSet.size()); - Collection<RoleInstance> clusterNodes = getLiveNodes().values(); + Collection<RoleInstance> clusterNodes = getLiveContainers().values(); for (RoleInstance node : clusterNodes) { if (uuidSet.contains(node.id)) { @@ -1080,7 +1064,7 @@ public class AppState { */ public synchronized List<RoleInstance> enumLiveNodesInRole(String role) { List<RoleInstance> nodes = new ArrayList<RoleInstance>(); - Collection<RoleInstance> allRoleInstances = getLiveNodes().values(); + Collection<RoleInstance> allRoleInstances = getLiveContainers().values(); for (RoleInstance node : allRoleInstances) { if (role.isEmpty() || role.equals(node.role)) { nodes.add(node); @@ -1117,7 +1101,7 @@ public class AppState { */ private synchronized Map<String, List<String>> createRoleToInstanceMap() { Map<String, List<String>> map = new HashMap<String, List<String>>(); - for (RoleInstance node : getLiveNodes().values()) { + for (RoleInstance node : getLiveContainers().values()) { List<String> containers = map.get(node.role); if (containers == null) { containers = new ArrayList<String>(); @@ -1136,7 +1120,7 @@ public class AppState { */ public synchronized Map<String, Map<String, ClusterNode>> createRoleToClusterNodeMap() { Map<String, Map<String, ClusterNode>> map = new HashMap<>(); - for (RoleInstance node : getLiveNodes().values()) { + for (RoleInstance node : getLiveContainers().values()) { Map<String, ClusterNode> containers = map.get(node.role); if (containers == null) { @@ -1160,7 +1144,7 @@ public class AppState { instance.state = STATE_SUBMITTED; instance.container = container; instance.createTime = now(); - getStartingNodes().put(container.getId(), instance); + getStartingContainers().put(container.getId(), instance); putOwnedContainer(container.getId(), instance); roleHistory.onContainerStartSubmitted(container, instance); } @@ -1354,7 +1338,7 @@ public class AppState { throw new RuntimeException( "Unknown role for node " + node); } - getLiveNodes().put(node.getContainerId(), node); + getLiveContainers().put(node.getContainerId(), node); //tell role history roleHistory.onContainerStarted(container); } @@ -1395,7 +1379,7 @@ public class AppState { instance); } instance.startTime = now(); - RoleInstance starting = getStartingNodes().remove(containerId); + RoleInstance starting = getStartingContainers().remove(containerId); if (null == starting) { throw new YarnRuntimeException( "Container "+ containerId +"%s is already started"); @@ -1423,7 +1407,7 @@ public class AppState { removeOwnedContainer(containerId); incFailedCountainerCount(); incStartFailedCountainerCount(); - RoleInstance instance = getStartingNodes().remove(containerId); + RoleInstance instance = getStartingContainers().remove(containerId); if (null != instance) { RoleStatus roleStatus = lookupRoleStatus(instance.roleId); String text; @@ -1434,7 +1418,7 @@ public class AppState { } instance.diagnostics = text; roleStatus.noteFailed(true, text, ContainerOutcome.Failed); - getFailedNodes().put(containerId, instance); + getFailedContainers().put(containerId, instance); roleHistory.onNodeManagerContainerStartFailed(instance.container); } } @@ -1547,11 +1531,11 @@ public class AppState { if (roleInstance != null) { //it was active, move it to failed incFailedCountainerCount(); - failedNodes.put(containerId, roleInstance); + failedContainers.put(containerId, roleInstance); } else { // the container may have been noted as failed already, so look // it up - roleInstance = failedNodes.get(containerId); + roleInstance = failedContainers.get(containerId); } if (roleInstance != null) { int roleId = roleInstance.roleId; @@ -1605,12 +1589,12 @@ public class AppState { //remove the node ContainerId id = status.getContainerId(); log.info("Removing node ID {}", id); - RoleInstance node = getLiveNodes().remove(id); + RoleInstance node = getLiveContainers().remove(id); if (node != null) { node.state = STATE_DESTROYED; node.exitCode = exitStatus; node.diagnostics = status.getDiagnostics(); - getCompletedNodes().put(id, node); + getCompletedContainers().put(id, node); result.roleInstance = node; } else { // not in the list @@ -1624,7 +1608,7 @@ public class AppState { // finally, verify the node doesn't exist any more assert !containersBeingReleased.containsKey( containerId) : "container still in release queue"; - assert !getLiveNodes().containsKey( + assert !getLiveContainers().containsKey( containerId) : " container still in live nodes"; assert getOwnedContainer(containerId) == null : "Container still in active container list";
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3aeab9ca/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ProviderAppState.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ProviderAppState.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ProviderAppState.java index 0508579..82b2f2a 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ProviderAppState.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ProviderAppState.java @@ -25,6 +25,7 @@ import org.apache.slider.api.ClusterDescription; import org.apache.slider.api.ClusterNode; import org.apache.slider.api.types.ApplicationLivenessInformation; import org.apache.slider.api.types.ComponentInformation; +import org.apache.slider.api.types.NodeInformation; import org.apache.slider.core.conf.AggregateConf; import org.apache.slider.core.conf.ConfTreeOperations; import org.apache.slider.core.exceptions.NoSuchNodeException; @@ -118,13 +119,13 @@ public class ProviderAppState implements StateAccessForProviders { @Override - public Map<ContainerId, RoleInstance> getFailedNodes() { - return appState.getFailedNodes(); + public Map<ContainerId, RoleInstance> getFailedContainers() { + return appState.getFailedContainers(); } @Override - public Map<ContainerId, RoleInstance> getLiveNodes() { - return appState.getLiveNodes(); + public Map<ContainerId, RoleInstance> getLiveContainers() { + return appState.getLiveContainers(); } @Override @@ -249,10 +250,10 @@ public class ProviderAppState implements StateAccessForProviders { } @Override - public List<RoleInstance> enumLiveNodesInRole(String role) { - List<RoleInstance> nodes = new ArrayList<RoleInstance>(); + public List<RoleInstance> enumLiveInstancesInRole(String role) { + List<RoleInstance> nodes = new ArrayList<>(); Collection<RoleInstance> allRoleInstances = cloneLiveContainerInfoList(); - getLiveNodes().values(); + getLiveContainers().values(); for (RoleInstance node : allRoleInstances) { if (role.isEmpty() || role.equals(node.role)) { nodes.add(node); @@ -265,8 +266,7 @@ public class ProviderAppState implements StateAccessForProviders { public List<RoleInstance> lookupRoleContainers(String component) { RoleStatus roleStatus = lookupRoleStatus(component); List<RoleInstance> ownedContainerList = cloneOwnedContainerList(); - List<RoleInstance> matching = - new ArrayList<>(ownedContainerList.size()); + List<RoleInstance> matching = new ArrayList<>(ownedContainerList.size()); int roleId = roleStatus.getPriority(); for (RoleInstance instance : ownedContainerList) { if (instance.roleId == roleId) { @@ -286,7 +286,16 @@ public class ProviderAppState implements StateAccessForProviders { info.containers.add(container.id); } return info; + } + @Override + public Map<String, NodeInformation> getNodeInformationSnapshot() { + return appState.getRoleHistory().getNodeInformationSnapshot(); + } + + @Override + public NodeInformation getNodeInformation(String hostname) { + return appState.getRoleHistory().getNodeInformation(hostname); } } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3aeab9ca/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java index f1e26bb..c22d517 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java @@ -26,9 +26,14 @@ import org.apache.hadoop.yarn.api.records.NodeReport; import org.apache.hadoop.yarn.api.records.NodeState; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.client.api.AMRMClient; +import org.apache.slider.api.types.NodeInformation; import org.apache.slider.common.tools.SliderUtils; import org.apache.slider.core.exceptions.BadConfigException; import org.apache.slider.providers.ProviderRole; +import org.apache.slider.server.appmaster.management.BoolMetric; +import org.apache.slider.server.appmaster.management.LongGauge; +import org.apache.slider.server.appmaster.management.MetricsAndMonitoring; +import org.apache.slider.server.appmaster.management.Timestamp; import org.apache.slider.server.appmaster.operations.AbstractRMOperation; import org.apache.slider.server.avro.LoadedRoleHistory; import org.apache.slider.server.avro.NodeEntryRecord; @@ -69,18 +74,25 @@ public class RoleHistory { private long startTime; /** Time when saved */ - private long saveTime; + private final Timestamp saveTime = new Timestamp(0); /** If the history was loaded, the time at which the history was saved */ - private long thawedDataTime; + private final Timestamp thawedDataTime = new Timestamp(0); private NodeMap nodemap; private int roleSize; - private boolean dirty; + private final BoolMetric dirty = new BoolMetric(false); private FileSystem filesystem; private Path historyPath; private RoleHistoryWriter historyWriter = new RoleHistoryWriter(); + /** + * When were the nodes updated in a {@link #onNodesUpdated(List)} call. + * If zero: never + */ + private final Timestamp nodesUpdatedTime = new Timestamp(0); + private final BoolMetric nodeUpdateReceived = new BoolMetric(false); + private OutstandingRequestTracker outstandingRequests = new OutstandingRequestTracker(); @@ -97,8 +109,7 @@ public class RoleHistory { private Set<String> failedNodes = new HashSet<>(); - public RoleHistory(List<ProviderRole> providerRoles) throws - BadConfigException { + public RoleHistory(List<ProviderRole> providerRoles) throws BadConfigException { this.providerRoles = providerRoles; roleSize = providerRoles.size(); reset(); @@ -106,15 +117,16 @@ public class RoleHistory { /** * Reset the variables -this does not adjust the fixed attributes - * of the history + * of the history, but the nodemap and failed node map are cleared. */ protected synchronized void reset() throws BadConfigException { nodemap = new NodeMap(roleSize); + failedNodes = new HashSet<>(); resetAvailableNodeLists(); outstandingRequests = new OutstandingRequestTracker(); - + Map<Integer, RoleStatus> roleStats = new HashMap<>(); for (ProviderRole providerRole : providerRoles) { checkProviderRole(roleStats, providerRole); @@ -122,6 +134,18 @@ public class RoleHistory { } /** + * Register all metrics with the metrics infra + * @param metrics metrics + */ + public void register(MetricsAndMonitoring metrics) { + metrics.register(RoleHistory.class, dirty, "dirty"); + metrics.register(RoleHistory.class, nodesUpdatedTime, "nodes-updated.time"); + metrics.register(RoleHistory.class, nodeUpdateReceived, "nodes-updated.flag"); + metrics.register(RoleHistory.class, thawedDataTime, "thawed.time"); + metrics.register(RoleHistory.class, saveTime, "saved.time"); + } + + /** * safety check: make sure the provider role is unique amongst * the role stats...which is extended with the new role * @param roleStats role stats @@ -145,7 +169,6 @@ public class RoleHistory { roleStats.put(index, new RoleStatus(providerRole)); } - /** * Add a new provider role to the map * @param providerRole new provider role @@ -187,8 +210,7 @@ public class RoleHistory { } /** - * Reset the variables -this does not adjust the fixed attributes - * of the history. + * Prepare the history for re-reading its state. * <p> * This intended for use by the RoleWriter logic. * @throws BadConfigException if there is a problem rebuilding the state @@ -243,21 +265,20 @@ public class RoleHistory { return discarded; } - public synchronized long getStartTime() { return startTime; } public synchronized long getSaveTime() { - return saveTime; + return saveTime.get(); } public long getThawedDataTime() { - return thawedDataTime; + return thawedDataTime.get(); } public void setThawedDataTime(long thawedDataTime) { - this.thawedDataTime = thawedDataTime; + this.thawedDataTime.set(thawedDataTime); } public synchronized int getRoleSize() { @@ -273,11 +294,11 @@ public class RoleHistory { } public synchronized boolean isDirty() { - return dirty; + return dirty.get(); } public synchronized void setDirty(boolean dirty) { - this.dirty = dirty; + this.dirty.set(dirty); } /** @@ -285,8 +306,8 @@ public class RoleHistory { * @param timestamp timestamp -updates the savetime field */ public synchronized void saved(long timestamp) { - dirty = false; - saveTime = timestamp; + setDirty(false); + saveTime.set(timestamp); } /** @@ -299,6 +320,29 @@ public class RoleHistory { } /** + * Get snapshot of the node map + * @return a snapshot of the current node state + */ + public Map<String, NodeInformation> getNodeInformationSnapshot() { + NodeMap map = cloneNodemap(); + Map<String, NodeInformation> result = new HashMap<>(map.size()); + for (Map.Entry<String, NodeInstance> entry : map.entrySet()) { + result.put(entry.getKey(), entry.getValue().serialize()); + } + return result; + } + + /** + * Get the information on a node + * @param hostname hostname + * @return the information about that host, or null if there is none + */ + public NodeInformation getNodeInformation(String hostname) { + NodeInstance nodeInstance = nodemap.get(hostname); + return nodeInstance != null ? nodeInstance.serialize() : null; + } + + /** * Get the node instance for the specific node -creating it if needed * @param hostname node address * @return the instance @@ -379,8 +423,7 @@ public class RoleHistory { */ public synchronized Path saveHistoryIfDirty() throws IOException { if (isDirty()) { - long time = now(); - return saveHistory(time); + return saveHistory(now()); } else { return null; } @@ -403,7 +446,7 @@ public class RoleHistory { } /** - * Handler for bootstrap event + * Handler for bootstrap event: there was no history to thaw */ public void onBootstrap() { log.debug("Role history bootstrapped"); @@ -746,12 +789,22 @@ public class RoleHistory { } /** + * Get the last time the nodes were updated from YARN + * @return the update time or zero if never updated. + */ + public long getNodesUpdatedTime() { + return nodesUpdatedTime.get(); + } + + /** * Update failedNodes and nodemap based on the node state * * @param updatedNodes list of updated nodes */ public synchronized void onNodesUpdated(List<NodeReport> updatedNodes) { log.debug("Updating {} nodes", updatedNodes.size()); + nodesUpdatedTime.set(now()); + nodeUpdateReceived.set(true); for (NodeReport updatedNode : updatedNodes) { String hostname = updatedNode.getNodeId() == null ? null http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3aeab9ca/slider-core/src/main/java/org/apache/slider/server/appmaster/state/StateAccessForProviders.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/StateAccessForProviders.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/StateAccessForProviders.java index 56c7cac..2fc00b2 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/StateAccessForProviders.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/StateAccessForProviders.java @@ -26,6 +26,7 @@ import org.apache.slider.api.ClusterNode; import org.apache.slider.api.StatusKeys; import org.apache.slider.api.types.ApplicationLivenessInformation; import org.apache.slider.api.types.ComponentInformation; +import org.apache.slider.api.types.NodeInformation; import org.apache.slider.core.conf.AggregateConf; import org.apache.slider.core.conf.ConfTreeOperations; import org.apache.slider.core.exceptions.NoSuchNodeException; @@ -87,17 +88,17 @@ public interface StateAccessForProviders { List<String> listConfigSets(); /** - * Get a map of all the failed nodes - * @return map of recorded failed notes + * Get a map of all the failed containers + * @return map of recorded failed containers */ - Map<ContainerId, RoleInstance> getFailedNodes(); + Map<ContainerId, RoleInstance> getFailedContainers(); /** - * Get the live nodes. + * Get the live containers. * * @return the live nodes */ - Map<ContainerId, RoleInstance> getLiveNodes(); + Map<ContainerId, RoleInstance> getLiveContainers(); /** * Get the current cluster description @@ -268,11 +269,11 @@ public interface StateAccessForProviders { Map<String, Map<String, ClusterNode>> getRoleClusterNodeMapping(); /** - * Enum all nodes by role. + * Enum all role instances by role. * @param role role, or "" for all roles - * @return a list of nodes, may be empty + * @return a list of instances, may be empty */ - List<RoleInstance> enumLiveNodesInRole(String role); + List<RoleInstance> enumLiveInstancesInRole(String role); /** * Look up all containers of a specific component name @@ -287,4 +288,19 @@ public interface StateAccessForProviders { * @return a structure describing the component. */ ComponentInformation getComponentInformation(String component); + + + /** + * Get a clone of the nodemap. + * The instances inside are not cloned + * @return a possibly empty map of hostname top info + */ + Map<String, NodeInformation> getNodeInformationSnapshot(); + + /** + * get information on a node + * @param hostname hostname to look up + * @return the information, or null if there is no information held. + */ + NodeInformation getNodeInformation(String hostname); } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3aeab9ca/slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApiImpl.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApiImpl.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApiImpl.java index 0730a21..a0fe310 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApiImpl.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApiImpl.java @@ -89,8 +89,7 @@ public class WebAppApiImpl implements WebAppApi { @Override public Map<String,RoleStatus> getRoleStatusByName() { List<RoleStatus> roleStatuses = appState.cloneRoleStatusList(); - Map<String, RoleStatus> map = - new TreeMap<String, RoleStatus>(); + Map<String, RoleStatus> map = new TreeMap<>(); for (RoleStatus status : roleStatuses) { map.put(status.getName(), status); } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3aeab9ca/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/RestPaths.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/RestPaths.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/RestPaths.java index 06b7ba2..424107c 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/RestPaths.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/RestPaths.java @@ -116,7 +116,6 @@ public class RestPaths { */ public static final String SYSTEM_THREADS = SYSTEM + "/threads"; - /** * application subpath */ @@ -134,7 +133,7 @@ public class RestPaths { public static final String LIVE_RESOURCES = "/live/resources"; public static final String LIVE_CONTAINERS = "/live/containers"; public static final String LIVE_COMPONENTS = "/live/components"; - public static final String LIVE_NODES = "/live/"; + public static final String LIVE_NODES = "/live/nodes"; public static final String LIVE_LIVENESS = "/live/liveness"; public static final String LIVE_STATISTICS = "/live/statistics"; public static final String MODEL = "/model"; http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3aeab9ca/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/ApplicationResouceContentCacheFactory.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/ApplicationResouceContentCacheFactory.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/ApplicationResouceContentCacheFactory.java index e7b8fc7..2facf16 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/ApplicationResouceContentCacheFactory.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/ApplicationResouceContentCacheFactory.java @@ -18,10 +18,6 @@ package org.apache.slider.server.appmaster.web.rest.application; -import org.apache.slider.api.types.ComponentInformation; -import org.apache.slider.api.types.ContainerInformation; -import org.apache.slider.core.conf.AggregateConf; -import org.apache.slider.core.conf.ConfTree; import org.apache.slider.server.appmaster.state.StateAccessForProviders; import org.apache.slider.server.appmaster.web.rest.application.resources.AggregateModelRefresher; import org.apache.slider.server.appmaster.web.rest.application.resources.AppconfRefresher; @@ -29,63 +25,39 @@ import org.apache.slider.server.appmaster.web.rest.application.resources.CachedC import org.apache.slider.server.appmaster.web.rest.application.resources.ContentCache; import org.apache.slider.server.appmaster.web.rest.application.resources.LiveComponentsRefresher; import org.apache.slider.server.appmaster.web.rest.application.resources.LiveContainersRefresher; +import org.apache.slider.server.appmaster.web.rest.application.resources.LiveNodesRefresher; import org.apache.slider.server.appmaster.web.rest.application.resources.LiveResourcesRefresher; import org.apache.slider.server.appmaster.web.rest.application.resources.LiveStatisticsRefresher; -import java.util.Map; - -import static org.apache.slider.server.appmaster.web.rest.RestPaths.LIVE_COMPONENTS; -import static org.apache.slider.server.appmaster.web.rest.RestPaths.LIVE_CONTAINERS; -import static org.apache.slider.server.appmaster.web.rest.RestPaths.LIVE_RESOURCES; -import static org.apache.slider.server.appmaster.web.rest.RestPaths.LIVE_STATISTICS; -import static org.apache.slider.server.appmaster.web.rest.RestPaths.MODEL_DESIRED; -import static org.apache.slider.server.appmaster.web.rest.RestPaths.MODEL_DESIRED_APPCONF; -import static org.apache.slider.server.appmaster.web.rest.RestPaths.MODEL_DESIRED_RESOURCES; -import static org.apache.slider.server.appmaster.web.rest.RestPaths.MODEL_RESOLVED; -import static org.apache.slider.server.appmaster.web.rest.RestPaths.MODEL_RESOLVED_APPCONF; -import static org.apache.slider.server.appmaster.web.rest.RestPaths.MODEL_RESOLVED_RESOURCES; +import static org.apache.slider.server.appmaster.web.rest.RestPaths.*; public class ApplicationResouceContentCacheFactory { public static final int LIFESPAN = 500; /** * Build the content cache - * @param cache cache to construct * @param state state view */ public static ContentCache createContentCache( StateAccessForProviders state) { ContentCache cache = new ContentCache(); - cache.put(LIVE_RESOURCES, - new CachedContent<ConfTree>(LIFESPAN, - new LiveResourcesRefresher(state))); - cache.put(LIVE_CONTAINERS, - new CachedContent<Map<String, ContainerInformation>>(LIFESPAN, - new LiveContainersRefresher(state))); - cache.put(LIVE_COMPONENTS, - new CachedContent<Map<String, ComponentInformation>>(LIFESPAN, - new LiveComponentsRefresher(state))); + cache.put(LIVE_RESOURCES, new CachedContent<>(LIFESPAN, new LiveResourcesRefresher(state))); + cache.put(LIVE_CONTAINERS, new CachedContent<>(LIFESPAN, new LiveContainersRefresher(state))); + cache.put(LIVE_COMPONENTS, new CachedContent<>(LIFESPAN, new LiveComponentsRefresher(state))); + cache.put(LIVE_NODES, new CachedContent<>(LIFESPAN, new LiveNodesRefresher(state))); cache.put(MODEL_DESIRED, - new CachedContent<AggregateConf>(LIFESPAN, - new AggregateModelRefresher(state, false))); + new CachedContent<>(LIFESPAN, new AggregateModelRefresher(state, false))); cache.put(MODEL_RESOLVED, - new CachedContent<AggregateConf>(LIFESPAN, - new AggregateModelRefresher(state, true))); + new CachedContent<>(LIFESPAN, new AggregateModelRefresher(state, true))); cache.put(MODEL_RESOLVED_APPCONF, - new CachedContent<ConfTree>(LIFESPAN, - new AppconfRefresher(state, false, false))); + new CachedContent<>(LIFESPAN, new AppconfRefresher(state, false, false))); cache.put(MODEL_RESOLVED_RESOURCES, - new CachedContent<ConfTree>(LIFESPAN, - new AppconfRefresher(state, false, true))); + new CachedContent<>(LIFESPAN, new AppconfRefresher(state, false, true))); cache.put(MODEL_DESIRED_APPCONF, - new CachedContent<ConfTree>(LIFESPAN, - new AppconfRefresher(state, true, false))); + new CachedContent<>(LIFESPAN, new AppconfRefresher(state, true, false))); cache.put(MODEL_DESIRED_RESOURCES, - new CachedContent<ConfTree>(LIFESPAN, - new AppconfRefresher(state, true, true))); - cache.put(LIVE_STATISTICS, - new CachedContent<Map<String, Integer>>(LIFESPAN, - new LiveStatisticsRefresher(state))); + new CachedContent<>(LIFESPAN, new AppconfRefresher(state, true, true))); + cache.put(LIVE_STATISTICS, new CachedContent<>(LIFESPAN, new LiveStatisticsRefresher(state))); return cache; } } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3aeab9ca/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/ApplicationResource.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/ApplicationResource.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/ApplicationResource.java index 383bc5a..1b54a31 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/ApplicationResource.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/ApplicationResource.java @@ -25,6 +25,7 @@ import org.apache.hadoop.yarn.webapp.NotFoundException; import org.apache.slider.api.types.ApplicationLivenessInformation; import org.apache.slider.api.types.ComponentInformation; import org.apache.slider.api.types.ContainerInformation; +import org.apache.slider.api.types.NodeInformation; import org.apache.slider.core.conf.AggregateConf; import org.apache.slider.core.conf.ConfTree; import org.apache.slider.core.exceptions.NoSuchNodeException; @@ -285,8 +286,7 @@ public class ApplicationResource extends AbstractSliderResource { public Map<String, ComponentInformation> getLiveComponents() { markGet(SLIDER_SUBPATH_APPLICATION, LIVE_COMPONENTS); try { - return (Map<String, ComponentInformation>) cache.lookup( - LIVE_COMPONENTS); + return (Map<String, ComponentInformation>) cache.lookup(LIVE_COMPONENTS); } catch (Exception e) { throw buildException(LIVE_COMPONENTS, e); } @@ -346,6 +346,38 @@ TODO: decide what structure to return here, then implement } */ + + @GET + @Path(LIVE_NODES) + @Produces({APPLICATION_JSON}) + public Map<String, NodeInformation> getLiveNodes() { + markGet(SLIDER_SUBPATH_APPLICATION, LIVE_COMPONENTS); + try { + return (Map<String, NodeInformation>) cache.lookup(LIVE_NODES); + } catch (Exception e) { + throw buildException(LIVE_COMPONENTS, e); + } + } + + @GET + @Path(LIVE_NODES + "/{node}") + @Produces({APPLICATION_JSON}) + public NodeInformation getLiveNode(@PathParam("node") String node) { + markGet(SLIDER_SUBPATH_APPLICATION, LIVE_COMPONENTS); + try { + NodeInformation ni = state.getNodeInformation(node); + if (ni != null) { + return ni; + } else { + throw new NotFoundException("Unknown node: " + node); + } + } catch (NotFoundException e) { + throw e; + } catch (Exception e) { + throw buildException(LIVE_CONTAINERS + "/" + node, e); + } + } + /** * Statistics of the application * @return snapshot statistics http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3aeab9ca/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/LiveNodesRefresher.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/LiveNodesRefresher.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/LiveNodesRefresher.java new file mode 100644 index 0000000..d4ab8fe --- /dev/null +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/LiveNodesRefresher.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.server.appmaster.web.rest.application.resources; + +import org.apache.slider.api.types.NodeInformation; +import org.apache.slider.server.appmaster.state.StateAccessForProviders; + +import java.util.Map; + +/** + * Update the live nodes map + */ +public class LiveNodesRefresher + implements ResourceRefresher<Map<String, NodeInformation>> { + + private final StateAccessForProviders state; + + public LiveNodesRefresher(StateAccessForProviders state) { + this.state = state; + } + + @Override + public Map<String, NodeInformation> refresh() { + return state.getNodeInformationSnapshot(); + } +} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3aeab9ca/slider-core/src/main/proto/SliderClusterMessages.proto ---------------------------------------------------------------------- diff --git a/slider-core/src/main/proto/SliderClusterMessages.proto b/slider-core/src/main/proto/SliderClusterMessages.proto index 8287f36..50c10e4 100644 --- a/slider-core/src/main/proto/SliderClusterMessages.proto +++ b/slider-core/src/main/proto/SliderClusterMessages.proto @@ -384,3 +384,15 @@ message GetCertificateStoreRequestProto { message GetCertificateStoreResponseProto { required bytes store = 1; } + +message GetLiveNodesRequestProto { +} + +message GetLiveNodesResponseProto { + repeated string names = 1; + repeated NodeInformationProto nodes = 2; +} + +message GetLiveNodeRequestProto { + required string name = 1; +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3aeab9ca/slider-core/src/main/proto/SliderClusterProtocol.proto ---------------------------------------------------------------------- diff --git a/slider-core/src/main/proto/SliderClusterProtocol.proto b/slider-core/src/main/proto/SliderClusterProtocol.proto index aa59bb4..4221b1d 100644 --- a/slider-core/src/main/proto/SliderClusterProtocol.proto +++ b/slider-core/src/main/proto/SliderClusterProtocol.proto @@ -129,18 +129,24 @@ service SliderClusterProtocolPB { rpc getLivenessInformation(GetApplicationLivenessRequestProto) returns(ApplicationLivenessInformationProto); - + rpc getLiveContainers(GetLiveContainersRequestProto) returns(GetLiveContainersResponseProto); rpc getLiveContainer(GetLiveContainerRequestProto) returns(ContainerInformationProto); - + rpc getLiveComponents(GetLiveComponentsRequestProto) returns(GetLiveComponentsResponseProto); - + rpc getLiveComponent(GetLiveComponentRequestProto) returns(ComponentInformationProto); + + rpc getLiveNodes(GetLiveNodesRequestProto) + returns(GetLiveNodesResponseProto); + + rpc getLiveNode(GetLiveNodeRequestProto) + returns(NodeInformationProto); // AggregateConf getModelDesired() rpc getModelDesired(EmptyPayloadProto) http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3aeab9ca/slider-core/src/test/groovy/org/apache/slider/agent/rest/AbstractAppApiTestDelegates.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/rest/AbstractAppApiTestDelegates.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/rest/AbstractAppApiTestDelegates.groovy index 58eb49e..6d1bcfc 100644 --- a/slider-core/src/test/groovy/org/apache/slider/agent/rest/AbstractAppApiTestDelegates.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/agent/rest/AbstractAppApiTestDelegates.groovy @@ -24,8 +24,10 @@ import org.apache.slider.api.SliderApplicationApi import org.apache.slider.api.StateValues import org.apache.slider.api.types.ComponentInformation import org.apache.slider.api.types.ContainerInformation +import org.apache.slider.api.types.NodeInformation import org.apache.slider.core.conf.ConfTreeOperations import org.apache.slider.test.Outcome +import org.junit.Test import static org.apache.slider.api.ResourceKeys.* import static org.apache.slider.api.StatusKeys.* @@ -203,6 +205,17 @@ public abstract class AbstractAppApiTestDelegates extends AbstractRestTestDelega assert !liveness.requestsOutstanding } + + public void testListNodes() throws Throwable { + describe "Node listing via $appAPI" + def liveNodes = appAPI.liveNodes + assert liveNodes.size() > 0 + def h = liveNodes.keySet()[0]; + def localhost = appAPI.getLiveNode(h) + + + } + /** * Probe that spins until the liveness query fails * @param args argument map @@ -226,6 +239,7 @@ public abstract class AbstractAppApiTestDelegates extends AbstractRestTestDelega testLiveContainers(); testRESTModel() testAppLiveness() +// testListNodes(); } public void testFlexOperation() { @@ -263,9 +277,10 @@ public abstract class AbstractAppApiTestDelegates extends AbstractRestTestDelega String key = args["key"] String val = args["val"] def resolved = appAPI.getResolvedResources() - return Outcome.fromBool(resolved.get(key)==val) + return Outcome.fromBool(resolved.get(key) == val) } + /** * Get the resolved value and push that out as the new state * http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3aeab9ca/slider-core/src/test/groovy/org/apache/slider/agent/rest/IpcApiClientTestDelegates.java ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/rest/IpcApiClientTestDelegates.java b/slider-core/src/test/groovy/org/apache/slider/agent/rest/IpcApiClientTestDelegates.java index 9411c3c..f339f6d 100644 --- a/slider-core/src/test/groovy/org/apache/slider/agent/rest/IpcApiClientTestDelegates.java +++ b/slider-core/src/test/groovy/org/apache/slider/agent/rest/IpcApiClientTestDelegates.java @@ -25,5 +25,5 @@ public class IpcApiClientTestDelegates extends AbstractAppApiTestDelegates { public IpcApiClientTestDelegates(SliderApplicationApi appAPI) { super(true, appAPI); } - + } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3aeab9ca/slider-core/src/test/groovy/org/apache/slider/agent/rest/LowLevelRestTestDelegates.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/rest/LowLevelRestTestDelegates.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/rest/LowLevelRestTestDelegates.groovy index c0f123d..3d87c28 100644 --- a/slider-core/src/test/groovy/org/apache/slider/agent/rest/LowLevelRestTestDelegates.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/agent/rest/LowLevelRestTestDelegates.groovy @@ -59,14 +59,14 @@ class LowLevelRestTestDelegates extends AbstractRestTestDelegate { application = appendToURL(appmaster, SLIDER_PATH_APPLICATION) } - public void testCodahaleOperations() throws Throwable { describe "Codahale operations $this" getWebPage(appmaster) getWebPage(appmaster, SYSTEM_THREADS) getWebPage(appmaster, SYSTEM_HEALTHCHECK) getWebPage(appmaster, SYSTEM_PING) - getWebPage(appmaster, SYSTEM_METRICS_JSON) + def page = getWebPage(appmaster, SYSTEM_METRICS_JSON) + validateCodahaleJson(parseMetrics(page)) } public void logCodahaleMetrics() { @@ -78,7 +78,7 @@ class LowLevelRestTestDelegates extends AbstractRestTestDelegate { public void testMimeTypes() throws Throwable { describe "Mime Types $this" - HttpOperationResponse response= executeGet( + HttpOperationResponse response = executeGet( appendToURL(appmaster, SLIDER_PATH_APPLICATION, LIVE_RESOURCES)) response.headers.each { key, val -> log.info("$key $val")} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3aeab9ca/slider-core/src/test/groovy/org/apache/slider/agent/rest/TestStandaloneREST.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/rest/TestStandaloneREST.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/rest/TestStandaloneREST.groovy index 7e10d30..880d9ca 100644 --- a/slider-core/src/test/groovy/org/apache/slider/agent/rest/TestStandaloneREST.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/agent/rest/TestStandaloneREST.groovy @@ -18,16 +18,8 @@ package org.apache.slider.agent.rest -import com.sun.jersey.api.client.Client -import com.sun.jersey.api.client.config.ClientConfig -import com.sun.jersey.api.json.JSONConfiguration -import com.sun.jersey.client.apache.ApacheHttpClient -import com.sun.jersey.client.apache.ApacheHttpClientHandler -import com.sun.jersey.client.apache.config.DefaultApacheHttpClientConfig import groovy.transform.CompileStatic import groovy.util.logging.Slf4j -import org.apache.commons.httpclient.HttpClient -import org.apache.commons.httpclient.MultiThreadedHttpConnectionManager import org.apache.hadoop.registry.client.api.RegistryOperations import org.apache.hadoop.yarn.api.records.ApplicationReport import org.apache.slider.agent.AgentMiniClusterTestBase @@ -42,8 +34,7 @@ import org.apache.slider.core.restclient.HttpOperationResponse import org.apache.slider.server.appmaster.rpc.RpcBinder import org.junit.Test -import static org.apache.slider.server.appmaster.management.MetricsKeys.METRICS_LOGGING_ENABLED -import static org.apache.slider.server.appmaster.management.MetricsKeys.METRICS_LOGGING_LOG_INTERVAL +import static org.apache.slider.server.appmaster.management.MetricsKeys.* import static org.apache.slider.server.appmaster.web.rest.RestPaths.* @CompileStatic @@ -79,7 +70,6 @@ class TestStandaloneREST extends AgentMiniClusterTestBase { // set up url config to match initHttpTestSupport(launcher.configuration) - execOperation(WEB_STARTUP_TIME) { GET(directAM) } @@ -88,7 +78,7 @@ class TestStandaloneREST extends AgentMiniClusterTestBase { def metrics = GET(directAM, SYSTEM_METRICS) log.info metrics } - + GET(proxyAM) log.info GET(proxyAM, SYSTEM_PING) @@ -96,15 +86,21 @@ class TestStandaloneREST extends AgentMiniClusterTestBase { log.info GET(proxyAM, SYSTEM_HEALTHCHECK) log.info GET(proxyAM, SYSTEM_METRICS_JSON) - /* - Is the back door required? If so, don't test complex verbs via the proxy - */ + // using the metrics, await the first node status update + execOperation(WEB_STARTUP_TIME) { + def metrics = getMetrics(proxyAM) + if (!getGaugeAsBool(metrics, + "org.apache.slider.server.appmaster.state.RoleHistory.nodes-updated.flag", false)) { + throw new IOException("Nodes not updated in $metrics") + } else { + "true" + }; + } + // Is the back door required? If so, don't test complex verbs via the proxy def proxyComplexVerbs = !SliderXmlConfKeys.X_DEV_INSECURE_REQUIRED - /* - * Only do direct complex verbs if the no back door is needed, or if - * it is enabled - */ + // Only do direct complex verbs if the no back door is needed, or if + // it is enabled def directComplexVerbs = proxyComplexVerbs || SLIDER_CONFIG.getBoolean( SliderXmlConfKeys.X_DEV_INSECURE_WS, SliderXmlConfKeys.X_DEV_INSECURE_DEFAULT) @@ -117,14 +113,12 @@ class TestStandaloneREST extends AgentMiniClusterTestBase { log.info "Content type: ${response.contentType}" describe "proxied response headers from AM Web resources" - response = executeGet(appendToURL(proxyAM, - SLIDER_PATH_APPLICATION, LIVE_RESOURCES)) + response = executeGet(appendToURL(proxyAM, SLIDER_PATH_APPLICATION, LIVE_RESOURCES)) response.headers.each { key, val -> log.info("$key $val") } log.info "Content type: ${response.contentType}" - def ugiClient = createUGIJerseyClient(); - + describe "Proxy SliderRestClient Tests" RestAPIClientTestDelegates proxySliderRestAPI = new RestAPIClientTestDelegates(proxyAM, ugiClient, proxyComplexVerbs) @@ -134,8 +128,7 @@ class TestStandaloneREST extends AgentMiniClusterTestBase { RestAPIClientTestDelegates directSliderRestAPI = new RestAPIClientTestDelegates(directAM, ugiClient, directComplexVerbs) directSliderRestAPI.testSuiteAll() - - + describe "Proxy Jersey Tests" JerseyTestDelegates proxyJerseyTests = new JerseyTestDelegates(proxyAM, ugiClient, proxyComplexVerbs) @@ -143,14 +136,12 @@ class TestStandaloneREST extends AgentMiniClusterTestBase { describe "Direct Jersey Tests" - JerseyTestDelegates directJerseyTests = - new JerseyTestDelegates(directAM, ugiClient) + JerseyTestDelegates directJerseyTests = new JerseyTestDelegates(directAM, ugiClient) directJerseyTests.testSuiteAll() describe "Direct Tests" - LowLevelRestTestDelegates direct = - new LowLevelRestTestDelegates(directAM, directComplexVerbs) + LowLevelRestTestDelegates direct = new LowLevelRestTestDelegates(directAM, directComplexVerbs) direct.testSuiteAll() describe "Proxy Tests" @@ -176,13 +167,10 @@ class TestStandaloneREST extends AgentMiniClusterTestBase { describe( "IPC equivalent operations") def sliderClusterProtocol = RpcBinder.getProxy(conf, report, 1000) - SliderApplicationIpcClient ipcClient = - new SliderApplicationIpcClient(sliderClusterProtocol) - IpcApiClientTestDelegates ipcDelegates = - new IpcApiClientTestDelegates(ipcClient) + SliderApplicationIpcClient ipcClient = new SliderApplicationIpcClient(sliderClusterProtocol) + IpcApiClientTestDelegates ipcDelegates = new IpcApiClientTestDelegates(ipcClient) ipcDelegates.testSuiteAll() - - + // log the metrics to show what's up direct.logCodahaleMetrics(); @@ -193,23 +181,4 @@ class TestStandaloneREST extends AgentMiniClusterTestBase { } } - /** - * Create Jersey client with URL handling by way - * of the Apache HttpClient classes. - * @return a Jersey client - */ - public static Client createJerseyClientHttpClient() { - - def httpclient = new HttpClient(new MultiThreadedHttpConnectionManager()); - httpclient.httpConnectionManager.params.connectionTimeout = 10000; - ClientConfig clientConfig = new DefaultApacheHttpClientConfig(); - clientConfig.features[JSONConfiguration.FEATURE_POJO_MAPPING] = Boolean.TRUE; - - def handler = new ApacheHttpClientHandler(httpclient, clientConfig); - - def client = new ApacheHttpClient(handler) - client.followRedirects = true - return client; - } - } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3aeab9ca/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAppRestIntegration.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAppRestIntegration.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAppRestIntegration.groovy index 7838886..d36fdbc 100644 --- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAppRestIntegration.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAppRestIntegration.groovy @@ -50,7 +50,6 @@ class TestMockAppStateAppRestIntegration extends BaseMockAppStateTest implements public void testCachedIntDocument() throws Throwable { ContentCache cache = new ContentCache() - def refresher = new IntRefresher() assert 0 == refresher.count def entry = new CachedContentManagedTimer(refresher) @@ -103,8 +102,8 @@ class TestMockAppStateAppRestIntegration extends BaseMockAppStateTest implements new ApplicationResource(webAppApi) def containers = applicationResource.liveContainers assert containers.size() == instances.size() - } + /** * Get a state accessor for the appState field * @return something to hand down to refreshers and resources @@ -146,7 +145,7 @@ class TestMockAppStateAppRestIntegration extends BaseMockAppStateTest implements class CachedContentManagedTimer extends CachedContent { int time = 0; - + @Override protected long now() { return time++; http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3aeab9ca/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/monkey/TestMockMonkey.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/monkey/TestMockMonkey.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/monkey/TestMockMonkey.groovy index 82192b9..ec202b0 100644 --- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/monkey/TestMockMonkey.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/monkey/TestMockMonkey.groovy @@ -148,7 +148,7 @@ class TestMockMonkey extends BaseMockAppStateTest { public void testContainerKillerIgnoresAM() throws Throwable { addAppMastertoAppState() - assert 1 == appState.liveNodes.size() + assert 1 == appState.liveContainers.size() def chaos = new ChaosKillContainer(appState, queues, http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3aeab9ca/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/publisher/TestPublisherRestResources.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/publisher/TestPublisherRestResources.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/publisher/TestPublisherRestResources.groovy index 29f0510..e301e6f 100644 --- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/publisher/TestPublisherRestResources.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/publisher/TestPublisherRestResources.groovy @@ -107,7 +107,7 @@ class TestPublisherRestResources extends AgentTestBase { webResource = client.resource(sliderConfigset + "dummy-site"); - execOperation(30000) { + execOperation(WEB_STARTUP_TIME) { GET(sliderConfigset) } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3aeab9ca/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy b/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy index 9d14815..c5808f2 100644 --- a/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy @@ -24,6 +24,7 @@ import com.sun.jersey.api.client.config.DefaultClientConfig import com.sun.jersey.api.json.JSONConfiguration import com.sun.jersey.client.urlconnection.URLConnectionClientHandler import groovy.json.JsonOutput +import groovy.json.JsonSlurper import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import org.apache.commons.httpclient.HttpClient @@ -73,6 +74,7 @@ import javax.ws.rs.core.HttpHeaders import java.util.concurrent.TimeoutException import static Arguments.ARG_OPTION +import static org.apache.slider.server.appmaster.web.rest.RestPaths.SYSTEM_METRICS_JSON /** * Static utils for tests in this package and in other test projects. @@ -655,7 +657,6 @@ class SliderTestUtils extends Assert { clientConfig); } - /** * Create a jersey client config with the settings needed for tests * (e.g. POJO mappings) @@ -707,7 +708,7 @@ class SliderTestUtils extends Assert { HttpCacheHeaders.HTTP_HEADER_CACHE_CONTROL_NONE } -/** + /** * Assert that a service operation succeeded * @param service service */ @@ -738,6 +739,7 @@ class SliderTestUtils extends Assert { int actual = instances != null ? instances.size() : 0 return actual } + /** * Exec a set of commands, wait a few seconds for it to finish. * @param status code @@ -752,11 +754,12 @@ class SliderTestUtils extends Assert { assert status == exitCode return process } + /** - * Exec a set of commands, wait a few seconds for it to finish. - * @param commands - * @return - */ + * Exec a set of commands, wait a few seconds for it to finish. + * @param commands + * @return + */ public static ForkedProcessService exec(List<String> commands) { ForkedProcessService process; process = new ForkedProcessService( @@ -1380,4 +1383,42 @@ class SliderTestUtils extends Assert { assert list.size() == entries.size() assert entries.containsAll(list) } + + public Map parseMetrics(String metrics) { + new JsonSlurper().parse(metrics.bytes) as Map + } + + public void validateCodahaleJson(Map metricsMap) { + assert metricsMap["version"] == "3.0.0" + assert metricsMap["gauges"] instanceof Map + assert metricsMap["histograms"] instanceof Map + assert metricsMap["timers"] instanceof Map + } + + public int getGaugeValue(Map metricsMap, String gauge, int defVal) { + def entry = metricsMap["gauges"][gauge] + if (entry != null) { + return entry["value"] as int + } else { + return defVal + } + } + + public boolean getGaugeAsBool(Map metricsMap, String gauge, boolean defVal) { + return 0 != getGaugeValue(metricsMap, gauge, defVal ? 1 : 0) + } + + /** + * Fetch and parse the JSON codahale metrics under a path + * @param baseUrl base path + * @return the fetch, parsed and partially validated JSON mapping + */ + public Map getMetrics(String baseUrl) { + def raw = GET(baseUrl, SYSTEM_METRICS_JSON) + def metrics = parseMetrics(raw) + validateCodahaleJson(metrics) + return metrics; + } + + }
