SLIDER-994 add "nodemap" command to get the (JSON) nodemap of the YARN cluster
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/eac0de9f Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/eac0de9f Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/eac0de9f Branch: refs/heads/feature/SLIDER-82-pass-3.1 Commit: eac0de9fcf7260ba8236825986108fc44548788a Parents: e9c5ebc Author: Steve Loughran <[email protected]> Authored: Fri Nov 20 17:45:30 2015 +0000 Committer: Steve Loughran <[email protected]> Committed: Fri Nov 20 17:45:30 2015 +0000 ---------------------------------------------------------------------- .../slider/api/types/NodeInformation.java | 23 +++++-- .../slider/api/types/NodeInformationList.java | 41 +++++++++++++ .../org/apache/slider/client/SliderClient.java | 63 ++++++++++++++++++-- .../apache/slider/client/SliderClientAPI.java | 12 ++++ .../slider/client/SliderYarnClientImpl.java | 46 ++++++++++++++ .../slider/common/params/ActionNodesArgs.java | 20 +++++++ .../apache/slider/common/tools/SliderUtils.java | 14 +++++ .../core/launch/JavaCommandLineBuilder.java | 2 - .../slider/core/persist/JsonSerDeser.java | 45 +++----------- .../slider/server/appmaster/state/AppState.java | 7 +-- .../server/appmaster/state/NodeInstance.java | 8 +-- .../slider/client/TestClientBadArgs.groovy | 16 +++-- .../client/TestSliderClientMethods.groovy | 3 +- .../client/TestUpgradeCommandOptions.groovy | 10 +--- .../slider/providers/agent/TestAgentEcho.groovy | 32 ++++++++++ 15 files changed, 270 insertions(+), 72 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/eac0de9f/slider-core/src/main/java/org/apache/slider/api/types/NodeInformation.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/api/types/NodeInformation.java b/slider-core/src/main/java/org/apache/slider/api/types/NodeInformation.java index edf7e21..e759bc9 100644 --- a/slider-core/src/main/java/org/apache/slider/api/types/NodeInformation.java +++ b/slider-core/src/main/java/org/apache/slider/api/types/NodeInformation.java @@ -33,12 +33,27 @@ import java.util.Map; @JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL) public class NodeInformation { - public String healthReport; public String hostname; - public String httpAddress; + public String state; public String labels; - public long lastUpdated; public String rackName; - public String state; + public String httpAddress; + public String healthReport; + public long lastUpdated; public Map<String, NodeEntryInformation> entries = new HashMap<>(); + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder( + "NodeInformation{"); + sb.append(", hostname='").append(hostname).append('\''); + sb.append(", state='").append(state).append('\''); + sb.append(", labels='").append(labels).append('\''); + sb.append(", rackName='").append(rackName).append('\''); + sb.append(", httpAddress='").append(httpAddress).append('\''); + sb.append(", healthReport='").append(healthReport).append('\''); + sb.append(", lastUpdated=").append(lastUpdated); + sb.append('}'); + return sb.toString(); + } } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/eac0de9f/slider-core/src/main/java/org/apache/slider/api/types/NodeInformationList.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/api/types/NodeInformationList.java b/slider-core/src/main/java/org/apache/slider/api/types/NodeInformationList.java new file mode 100644 index 0000000..741523e --- /dev/null +++ b/slider-core/src/main/java/org/apache/slider/api/types/NodeInformationList.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.api.types; + +import org.apache.slider.core.persist.JsonSerDeser; + +import java.util.ArrayList; +import java.util.Collection; + +public class NodeInformationList extends ArrayList<NodeInformation> { + public NodeInformationList() { + } + + public NodeInformationList(Collection<? extends NodeInformation> c) { + super(c); + } + + public NodeInformationList(int initialCapacity) { + super(initialCapacity); + } + + public static JsonSerDeser<NodeInformationList> createSerializer() { + return new JsonSerDeser<>(NodeInformationList.class); + } +} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/eac0de9f/slider-core/src/main/java/org/apache/slider/client/SliderClient.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java index bfec417..ca9bb12 100644 --- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java +++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java @@ -68,6 +68,7 @@ import org.apache.slider.api.SliderClusterProtocol; import org.apache.slider.api.StateValues; import org.apache.slider.api.proto.Messages; import org.apache.slider.api.types.ContainerInformation; +import org.apache.slider.api.types.NodeInformationList; import org.apache.slider.api.types.SliderInstanceDescription; import org.apache.slider.client.ipc.SliderClusterOperations; import org.apache.slider.common.Constants; @@ -91,6 +92,7 @@ import org.apache.slider.common.params.ActionKeytabArgs; import org.apache.slider.common.params.ActionKillContainerArgs; import org.apache.slider.common.params.ActionListArgs; import org.apache.slider.common.params.ActionLookupArgs; +import org.apache.slider.common.params.ActionNodesArgs; import org.apache.slider.common.params.ActionPackageArgs; import org.apache.slider.common.params.ActionRegistryArgs; import org.apache.slider.common.params.ActionResolveArgs; @@ -133,6 +135,7 @@ import org.apache.slider.core.main.RunService; import org.apache.slider.core.persist.AppDefinitionPersister; import org.apache.slider.core.persist.ApplicationReportSerDeser; import org.apache.slider.core.persist.ConfPersister; +import org.apache.slider.core.persist.JsonSerDeser; import org.apache.slider.core.persist.LockAcquireFailedException; import org.apache.slider.core.registry.SliderRegistryUtils; import org.apache.slider.core.registry.YarnAppListClient; @@ -166,7 +169,6 @@ import org.slf4j.LoggerFactory; import java.io.BufferedReader; import java.io.File; -import java.io.FilenameFilter; import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; @@ -333,7 +335,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe if (isUnset(action)) { throw new SliderException(EXIT_USAGE, serviceArgs.usage()); } - + int exitCode = EXIT_SUCCESS; String clusterName = serviceArgs.getClusterName(); // actions @@ -405,11 +407,15 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe case ACTION_LIST: exitCode = actionList(clusterName, serviceArgs.getActionListArgs()); break; - + case ACTION_LOOKUP: exitCode = actionLookup(serviceArgs.getActionLookupArgs()); break; + case ACTION_NODES: + exitCode = actionNodes("", serviceArgs.getActionNodesArgs()); + break; + case ACTION_PACKAGE: exitCode = actionPackage(serviceArgs.getActionPackageArgs()); break; @@ -2156,7 +2162,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe log.info("Using queue {} for the application instance.", amQueue); } - if (amQueue != null) { + if (isSet(amQueue)) { amLauncher.setQueue(amQueue); } @@ -4225,6 +4231,55 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe throw new UsageException("%s %s", errMsg, CommonArgs.usage(serviceArgs, actionName)); } + + /** + * List the nodes in the cluster, possibly filtering by node state or label. + * + * @param args argument list + * @return a possibly empty list of nodes in the cluster + * @throws IOException IO problems + * @throws YarnException YARN problems + */ + @Override + public NodeInformationList listYarnClusterNodes(ActionNodesArgs args) + throws YarnException, IOException { + return yarnClient.listNodes(args.label, args.healthy); + } + + /** + * List the nodes in the cluster, possibly filtering by node state or label. + * + * @param args argument list + * @return a possibly empty list of nodes in the cluster + * @throws IOException IO problems + * @throws YarnException YARN problems + */ + public NodeInformationList listInstanceNodes(ActionNodesArgs args) + throws YarnException, IOException { + return yarnClient.listNodes(args.label, args.healthy); + } + + /** + * List the nodes in the cluster, possibly filtering by node state or label. + * Prints them to stdout unless the args names a file instead. + * @param args argument list + * @throws IOException IO problems + * @throws YarnException YARN problems + */ + public int actionNodes(String instance, ActionNodesArgs args) throws YarnException, IOException { + + args.instance = instance; + NodeInformationList nodes = listYarnClusterNodes(args); + log.debug("Node listing for {} has {} nodes", args, nodes.size()); + JsonSerDeser<NodeInformationList> serDeser = NodeInformationList.createSerializer(); + if (args.outputFile != null) { + serDeser.save(nodes, args.outputFile); + } else { + println(serDeser.toJson(nodes)); + } + return 0; + } + } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/eac0de9f/slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java b/slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java index d87b121..5c5d96b 100644 --- a/slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java +++ b/slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java @@ -23,6 +23,7 @@ import org.apache.hadoop.service.Service; import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.slider.api.types.NodeInformationList; import org.apache.slider.api.types.SliderInstanceDescription; import org.apache.slider.common.params.AbstractClusterBuildingActionArgs; import org.apache.slider.common.params.ActionAMSuicideArgs; @@ -36,6 +37,7 @@ import org.apache.slider.common.params.ActionFreezeArgs; import org.apache.slider.common.params.ActionInstallKeytabArgs; import org.apache.slider.common.params.ActionInstallPackageArgs; import org.apache.slider.common.params.ActionKeytabArgs; +import org.apache.slider.common.params.ActionNodesArgs; import org.apache.slider.common.params.ActionPackageArgs; import org.apache.slider.common.params.ActionKillContainerArgs; import org.apache.slider.common.params.ActionListArgs; @@ -342,4 +344,14 @@ public interface SliderClientAPI extends Service { */ int actionDependency(ActionDependencyArgs dependencyArgs) throws IOException, YarnException; + + /** + * List the nodes + * @param args + * @return + * @throws YarnException + * @throws IOException + */ + NodeInformationList listYarnClusterNodes(ActionNodesArgs args) + throws YarnException, IOException; } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/eac0de9f/slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java b/slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java index a315345..85a582b 100644 --- a/slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java +++ b/slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java @@ -28,13 +28,18 @@ import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationRequest; import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationResponse; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; +import org.apache.hadoop.yarn.api.records.NodeReport; +import org.apache.hadoop.yarn.api.records.NodeState; import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.client.api.impl.YarnClientImpl; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.hadoop.yarn.util.Records; +import org.apache.slider.api.types.NodeInformation; +import org.apache.slider.api.types.NodeInformationList; import org.apache.slider.common.SliderKeys; +import org.apache.slider.common.params.ActionNodesArgs; import org.apache.slider.common.tools.CoreFileSystem; import org.apache.slider.common.tools.Duration; import org.apache.slider.common.tools.SliderFileSystem; @@ -361,4 +366,45 @@ public class SliderYarnClientImpl extends YarnClientImpl { log.debug("No match"); return null; } + + /** + * List the nodes in the cluster, possibly filtering by node state or label. + * + * @param label label to filter by -or "" for any + * @param live flag to request running nodes only + * @return a possibly empty list of nodes in the cluster + * @throws IOException IO problems + * @throws YarnException YARN problems + */ + public NodeInformationList listNodes(String label, boolean live) + throws IOException, YarnException { + Preconditions.checkArgument(label != null, "null label"); + NodeState[] states; + if (live) { + states = new NodeState[1]; + states[0] = NodeState.RUNNING; + } else { + states = new NodeState[0]; + } + List<NodeReport> reports = getNodeReports(states); + NodeInformationList results = new NodeInformationList(reports.size()); + for (NodeReport report : reports) { + if (live && report.getNodeState() != NodeState.RUNNING) { + continue; + } + if (!label.isEmpty() && !report.getNodeLabels().contains(label)) { + continue; + } + // build node info from report + NodeInformation info = new NodeInformation(); + info.hostname = report.getNodeId().getHost(); + info.healthReport = report.getHealthReport(); + info.httpAddress = report.getHttpAddress(); + info.labels = SliderUtils.extractNodeLabel(report); + info.rackName = report.getRackName(); + info.state = report.getNodeState().toString(); + results.add(info); + } + return results; + } } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/eac0de9f/slider-core/src/main/java/org/apache/slider/common/params/ActionNodesArgs.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/common/params/ActionNodesArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/ActionNodesArgs.java index bb214e1..5ddccf6 100644 --- a/slider-core/src/main/java/org/apache/slider/common/params/ActionNodesArgs.java +++ b/slider-core/src/main/java/org/apache/slider/common/params/ActionNodesArgs.java @@ -27,6 +27,11 @@ import java.io.File; commandDescription = SliderActions.DESCRIBE_ACTION_NODES) public class ActionNodesArgs extends AbstractActionArgs { + /** + * Instance for API use; on CLI the name is derived from {@link #getClusterName()}. + */ + public String instance; + @Override public String getActionName() { return SliderActions.ACTION_NODES; @@ -42,5 +47,20 @@ public class ActionNodesArgs extends AbstractActionArgs { @Parameter(names = {ARG_HEALTHY} ) public boolean healthy; + @Override + public int getMinParams() { + return 0; + } + @Override + public String toString() { + final StringBuilder sb = new StringBuilder( + "ActionNodesArgs{"); + sb.append("instance='").append(instance).append('\''); + sb.append(", outputFile=").append(outputFile); + sb.append(", label='").append(label).append('\''); + sb.append(", healthy=").append(healthy); + sb.append('}'); + return sb.toString(); + } } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/eac0de9f/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java index eb7a9d5..712eb75 100644 --- a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java +++ b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java @@ -47,6 +47,7 @@ import org.apache.hadoop.yarn.api.ApplicationConstants; import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.LocalResource; +import org.apache.hadoop.yarn.api.records.NodeReport; import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.client.api.AMRMClient; import org.apache.hadoop.yarn.conf.YarnConfiguration; @@ -1774,6 +1775,19 @@ public final class SliderUtils { return toTruncate.substring(0, maxSize - pad.length()).concat(pad); } + /** + * Get a string node label value from a node report + * @param report node report + * @return a single trimmed label or "" + */ + public static String extractNodeLabel(NodeReport report) { + Set<String> newlabels = report.getNodeLabels(); + if (newlabels != null && !newlabels.isEmpty()) { + return newlabels.iterator().next().trim(); + } else { + return ""; + } + } /** * Callable for async/scheduled halt http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/eac0de9f/slider-core/src/main/java/org/apache/slider/core/launch/JavaCommandLineBuilder.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/core/launch/JavaCommandLineBuilder.java b/slider-core/src/main/java/org/apache/slider/core/launch/JavaCommandLineBuilder.java index ccb610a..dcb322e 100644 --- a/slider-core/src/main/java/org/apache/slider/core/launch/JavaCommandLineBuilder.java +++ b/slider-core/src/main/java/org/apache/slider/core/launch/JavaCommandLineBuilder.java @@ -25,8 +25,6 @@ import org.apache.hadoop.yarn.api.ApplicationConstants; import org.apache.slider.common.tools.SliderUtils; import org.apache.slider.core.exceptions.BadConfigException; -import java.util.Map; - /** * Command line builder purely for the Java CLI. * Some of the <code>define</code> methods are designed to work with Hadoop tool and http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/eac0de9f/slider-core/src/main/java/org/apache/slider/core/persist/JsonSerDeser.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/core/persist/JsonSerDeser.java b/slider-core/src/main/java/org/apache/slider/core/persist/JsonSerDeser.java index c5908bb..4f60c06 100644 --- a/slider-core/src/main/java/org/apache/slider/core/persist/JsonSerDeser.java +++ b/slider-core/src/main/java/org/apache/slider/core/persist/JsonSerDeser.java @@ -91,10 +91,11 @@ public class JsonSerDeser<T> { */ public T fromFile(File jsonFile) throws IOException, JsonParseException, JsonMappingException { + File absoluteFile = jsonFile.getAbsoluteFile(); try { - return mapper.readValue(jsonFile, classType); + return mapper.readValue(absoluteFile, classType); } catch (IOException e) { - log.error("Exception while parsing json file {}: {}", jsonFile, e); + log.error("Exception while parsing json file {}", absoluteFile, e); throw e; } } @@ -106,7 +107,6 @@ public class JsonSerDeser<T> { * @throws IOException IO problems * @throws JsonMappingException failure to map from the JSON to this class */ -/* JDK7 public T fromResource(String resource) throws IOException, JsonParseException, JsonMappingException { try(InputStream resStream = this.getClass().getResourceAsStream(resource)) { @@ -115,34 +115,9 @@ public class JsonSerDeser<T> { } return (T) (mapper.readValue(resStream, classType)); } catch (IOException e) { - log.error("Exception while parsing json resource {}: {}", resource, e); + log.error("Exception while parsing json resource {}", resource, e); throw e; } - }*/ - - /** - * Convert from a JSON file - * @param resource input file - * @return the parsed JSON - * @throws IOException IO problems - * @throws JsonMappingException failure to map from the JSON to this class - */ - @SuppressWarnings("IOResourceOpenedButNotSafelyClosed") - public synchronized T fromResource(String resource) - throws IOException, JsonParseException, JsonMappingException { - InputStream resStream = null; - try { - resStream = this.getClass().getResourceAsStream(resource); - if (resStream == null) { - throw new FileNotFoundException(resource); - } - return (T) (mapper.readValue(resStream, classType)); - } catch (IOException e) { - log.error("Exception while parsing json resource {}: {}", resource, e); - throw e; - } finally { - IOUtils.closeStream(resStream); - } } /** @@ -155,7 +130,7 @@ public class JsonSerDeser<T> { try { return (T) (mapper.readValue(stream, classType)); } catch (IOException e) { - log.error("Exception while parsing json input stream: {}", e); + log.error("Exception while parsing json input stream", e); throw e; } finally { IOUtils.closeStream(stream); @@ -201,7 +176,7 @@ public class JsonSerDeser<T> { FSDataInputStream dataInputStream = fs.open(path); int count = dataInputStream.read(b); if (count != len) { - throw new EOFException("Read finished prematurely"); + throw new EOFException("Read of " + path +" finished prematurely"); } return fromBytes(b); } @@ -230,9 +205,9 @@ public class JsonSerDeser<T> { */ public void save(T instance, File file) throws IOException { - writeJsonAsBytes(instance, new FileOutputStream(file)); + writeJsonAsBytes(instance, new FileOutputStream(file.getAbsoluteFile())); } - + /** * Write the json as bytes -then close the file * @param dataOutputStream an outout stream that will always be closed @@ -251,7 +226,6 @@ public class JsonSerDeser<T> { } } - /** * Convert an object to a JSON string * @param instance instance to convert @@ -265,6 +239,5 @@ public class JsonSerDeser<T> { mapper.configure(SerializationConfig.Feature.INDENT_OUTPUT, true); return mapper.writeValueAsString(instance); } - - + } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/eac0de9f/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java index 49bc225..ee5d43d 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java @@ -688,8 +688,8 @@ public class AppState { instanceDefinition.resolve(); // force in the AM desired state values - ConfTreeOperations resources = - instanceDefinition.getResourceOperations(); + ConfTreeOperations resources = instanceDefinition.getResourceOperations(); + if (resources.getComponent(SliderKeys.COMPONENT_AM) != null) { resources.setComponentOpt( SliderKeys.COMPONENT_AM, COMPONENT_INSTANCES, "1"); @@ -706,8 +706,7 @@ public class AppState { internalsSnapshot.confTree); instanceDefinitionSnapshot.setName(instanceDefinition.getName()); - clusterStatusTemplate = - ClusterDescriptionOperations.buildFromInstanceDefinition( + clusterStatusTemplate = ClusterDescriptionOperations.buildFromInstanceDefinition( instanceDefinition); // Add the -site configuration properties http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/eac0de9f/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeInstance.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeInstance.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeInstance.java index 2b8f01c..8d63239 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeInstance.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeInstance.java @@ -31,7 +31,6 @@ import java.util.Comparator; import java.util.HashMap; import java.util.List; import java.util.ListIterator; -import java.util.Set; /** * A node instance -stores information about a node in the cluster. @@ -103,12 +102,7 @@ public class NodeInstance { boolean newUsable = !nodeState.isUnusable(); boolean nodeNowAvailable = oldStateUnusable && newUsable; String labels = this.nodeLabels; - Set<String> newlabels = report.getNodeLabels(); - if (newlabels != null && !newlabels.isEmpty()) { - nodeLabels = newlabels.iterator().next().trim(); - } else { - nodeLabels = ""; - } + nodeLabels = SliderUtils.extractNodeLabel(report); return nodeNowAvailable || newUsable && !this.nodeLabels.equals(labels); } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/eac0de9f/slider-core/src/test/groovy/org/apache/slider/client/TestClientBadArgs.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/client/TestClientBadArgs.groovy b/slider-core/src/test/groovy/org/apache/slider/client/TestClientBadArgs.groovy index 1b074dc..9b6c7dc 100644 --- a/slider-core/src/test/groovy/org/apache/slider/client/TestClientBadArgs.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/client/TestClientBadArgs.groovy @@ -22,14 +22,11 @@ import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import org.apache.hadoop.conf.Configuration import org.apache.hadoop.yarn.conf.YarnConfiguration -import org.apache.slider.common.params.ActionRegistryArgs import org.apache.slider.common.params.Arguments import org.apache.slider.common.params.SliderActions import org.apache.slider.core.exceptions.BadCommandArgumentsException import org.apache.slider.core.exceptions.ErrorStrings import org.apache.slider.core.exceptions.UsageException -import org.apache.slider.core.exceptions.BadConfigException -import org.apache.slider.core.main.ServiceLauncher import org.apache.slider.core.main.ServiceLauncherBaseTest import org.junit.Test @@ -39,9 +36,7 @@ import org.junit.Test @CompileStatic @Slf4j class TestClientBadArgs extends ServiceLauncherBaseTest { - - static String TEST_FILES = "./src/test/resources/org/apache/slider/providers/agent/tests/" - + @Test public void testNoAction() throws Throwable { launchExpectingException(SliderClient, @@ -244,4 +239,13 @@ class TestClientBadArgs extends ServiceLauncherBaseTest { assert exception instanceof BadCommandArgumentsException log.info(exception.toString()) } + + @Test + public void testNodesMissingFile() throws Throwable { + def exception = launchExpectingException(SliderClient, + createTestConfig(), + "after parameter --out", + [SliderActions.ACTION_NODES, Arguments.ARG_OUTPUT]) + assert exception instanceof BadCommandArgumentsException + } } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/eac0de9f/slider-core/src/test/groovy/org/apache/slider/client/TestSliderClientMethods.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/client/TestSliderClientMethods.groovy b/slider-core/src/test/groovy/org/apache/slider/client/TestSliderClientMethods.groovy index 168415c..e244923 100644 --- a/slider-core/src/test/groovy/org/apache/slider/client/TestSliderClientMethods.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/client/TestSliderClientMethods.groovy @@ -30,6 +30,7 @@ import org.apache.slider.core.exceptions.SliderException import org.apache.slider.core.launch.LaunchedApplication import org.apache.slider.core.main.ServiceLauncherBaseTest import org.apache.slider.core.persist.LockAcquireFailedException +import org.apache.slider.server.appmaster.model.mock.MockApplicationId import org.easymock.EasyMock import org.junit.Assert import org.junit.Test @@ -137,7 +138,7 @@ class TestSliderClientMethods extends ServiceLauncherBaseTest { AggregateConf instanceDefinition, boolean debugAM) throws YarnException, IOException { - return new LaunchedApplication(clustername, new SliderYarnClientImpl()); + return new LaunchedApplication(new MockApplicationId(1), new SliderYarnClientImpl()); } } } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/eac0de9f/slider-core/src/test/groovy/org/apache/slider/client/TestUpgradeCommandOptions.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/client/TestUpgradeCommandOptions.groovy b/slider-core/src/test/groovy/org/apache/slider/client/TestUpgradeCommandOptions.groovy index 8d5aef1..5295f9b 100644 --- a/slider-core/src/test/groovy/org/apache/slider/client/TestUpgradeCommandOptions.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/client/TestUpgradeCommandOptions.groovy @@ -82,10 +82,8 @@ class TestUpgradeCommandOptions extends AgentMiniClusterTestBase { new File(tmpDir, "resources.json").toURI() ]) fail("Upgrade command should have failed") - } catch (SliderException e) { - log.info(e.toString()) - assert e instanceof UnknownApplicationInstanceException - assert e.getMessage().contains("Unknown application instance") + } catch (UnknownApplicationInstanceException e) { + assertExceptionDetails(e, SliderExitCodes.EXIT_UNKNOWN_INSTANCE, "Unknown application instance") } } @@ -301,10 +299,6 @@ class TestUpgradeCommandOptions extends AgentMiniClusterTestBase { assert launcher.serviceExitCode == 0 } - private File getTempLocation () { - return new File(System.getProperty("user.dir") + "/target/_") - } - static class TestSliderClient extends SliderClient { public TestSliderClient() { super() http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/eac0de9f/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentEcho.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentEcho.groovy b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentEcho.groovy index 4ceeca0..23a7bbb 100644 --- a/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentEcho.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentEcho.groovy @@ -22,11 +22,14 @@ import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import org.apache.hadoop.yarn.conf.YarnConfiguration import org.apache.slider.api.ResourceKeys +import org.apache.slider.api.types.NodeInformationList import org.apache.slider.client.SliderClient import org.apache.slider.common.SliderExitCodes import org.apache.slider.common.SliderXmlConfKeys +import org.apache.slider.common.params.ActionNodesArgs import org.apache.slider.core.exceptions.BadClusterStateException import org.apache.slider.core.main.ServiceLauncher +import org.apache.slider.core.persist.JsonSerDeser import org.junit.Before import org.junit.Test @@ -124,6 +127,35 @@ class TestAgentEcho extends AgentTestBase { assertExceptionDetails(e, SliderExitCodes.EXIT_BAD_STATE, "negative") } + + runNodemapTests(sliderClient) + + } + + /** + * do some nodemap checks, currently cluster-wide + * @param sliderClient + */ + protected void runNodemapTests(SliderClient sliderClient) { + describe "slider nodes" + sliderClient.actionNodes("", new ActionNodesArgs()) + + def allNodes = sliderClient.listYarnClusterNodes(new ActionNodesArgs()).collect { it.httpAddress } + assert !allNodes.empty + + // healthy only + def healthyNodes = sliderClient.listYarnClusterNodes(new ActionNodesArgs(healthy: true)) + assert healthyNodes.collect { it.httpAddress }.containsAll(allNodes) + // look for an unknown label and expect none + def gpuNodes = sliderClient.listYarnClusterNodes(new ActionNodesArgs(label: "gpu")) + assert gpuNodes.empty + File t1 = createTempJsonFile() + sliderClient.actionNodes("", new ActionNodesArgs(outputFile: t1)) + assert t1.exists() + JsonSerDeser<NodeInformationList> serDeser = new JsonSerDeser<>(NodeInformationList.class); + NodeInformationList loaded = serDeser.fromFile(t1) + assert allNodes.containsAll(loaded.collect { it.httpAddress }) + } protected void validatePaths() {
