Repository: hadoop Updated Branches: refs/heads/trunk db1bba857 -> 78860372b
YARN-8134. Support specifying node resources in SLS. Contributed by Abhishek Modi. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/78860372 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/78860372 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/78860372 Branch: refs/heads/trunk Commit: 78860372bd8048168c6aa27a9526c40f5869cf2c Parents: db1bba8 Author: Inigo Goiri <[email protected]> Authored: Tue Apr 17 13:11:34 2018 -0700 Committer: Inigo Goiri <[email protected]> Committed: Tue Apr 17 13:11:34 2018 -0700 ---------------------------------------------------------------------- hadoop-tools/hadoop-sls/pom.xml | 1 + .../org/apache/hadoop/yarn/sls/SLSRunner.java | 35 ++++++++++++++------ .../apache/hadoop/yarn/sls/utils/SLSUtils.java | 24 +++++++++++--- .../hadoop/yarn/sls/utils/TestSLSUtils.java | 25 ++++++++++++++ .../test/resources/nodes-with-resources.json | 19 +++++++++++ 5 files changed, 89 insertions(+), 15 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/78860372/hadoop-tools/hadoop-sls/pom.xml ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-sls/pom.xml b/hadoop-tools/hadoop-sls/pom.xml index 10f6294..a42f8de 100644 --- a/hadoop-tools/hadoop-sls/pom.xml +++ b/hadoop-tools/hadoop-sls/pom.xml @@ -137,6 +137,7 @@ <exclude>src/test/resources/syn_stream.json</exclude> <exclude>src/test/resources/inputsls.json</exclude> <exclude>src/test/resources/nodes.json</exclude> + <exclude>src/test/resources/nodes-with-resources.json</exclude> <exclude>src/test/resources/exit-invariants.txt</exclude> <exclude>src/test/resources/ongoing-invariants.txt</exclude> </excludes> http://git-wip-us.apache.org/repos/asf/hadoop/blob/78860372/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java index 8a522fe..8504b9d 100644 --- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java +++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java @@ -292,21 +292,30 @@ public class SLSRunner extends Configured implements Tool { SLSConfiguration.NM_RESOURCE_UTILIZATION_RATIO, SLSConfiguration.NM_RESOURCE_UTILIZATION_RATIO_DEFAULT); // nm information (fetch from topology file, or from sls/rumen json file) - Set<String> nodeSet = new HashSet<String>(); + Map<String, Resource> nodeResourceMap = new HashMap<>(); + Set<? extends String> nodeSet; if (nodeFile.isEmpty()) { for (String inputTrace : inputTraces) { - switch (inputType) { case SLS: - nodeSet.addAll(SLSUtils.parseNodesFromSLSTrace(inputTrace)); + nodeSet = SLSUtils.parseNodesFromSLSTrace(inputTrace); + for (String node : nodeSet) { + nodeResourceMap.put(node, null); + } break; case RUMEN: - nodeSet.addAll(SLSUtils.parseNodesFromRumenTrace(inputTrace)); + nodeSet = SLSUtils.parseNodesFromRumenTrace(inputTrace); + for (String node : nodeSet) { + nodeResourceMap.put(node, null); + } break; case SYNTH: stjp = new SynthTraceJobProducer(getConf(), new Path(inputTraces[0])); - nodeSet.addAll(SLSUtils.generateNodes(stjp.getNumNodes(), - stjp.getNumNodes()/stjp.getNodesPerRack())); + nodeSet = SLSUtils.generateNodes(stjp.getNumNodes(), + stjp.getNumNodes()/stjp.getNodesPerRack()); + for (String node : nodeSet) { + nodeResourceMap.put(node, null); + } break; default: throw new YarnException("Input configuration not recognized, " @@ -314,20 +323,26 @@ public class SLSRunner extends Configured implements Tool { } } } else { - nodeSet.addAll(SLSUtils.parseNodesFromNodeFile(nodeFile)); + nodeResourceMap = SLSUtils.parseNodesFromNodeFile(nodeFile, + nodeManagerResource); } - if (nodeSet.size() == 0) { + if (nodeResourceMap.size() == 0) { throw new YarnException("No node! Please configure nodes."); } // create NM simulators Random random = new Random(); Set<String> rackSet = new HashSet<String>(); - for (String hostName : nodeSet) { + for (Map.Entry<String, Resource> entry : nodeResourceMap.entrySet()) { // we randomize the heartbeat start time from zero to 1 interval NMSimulator nm = new NMSimulator(); - nm.init(hostName, nodeManagerResource, random.nextInt(heartbeatInterval), + Resource nmResource = nodeManagerResource; + String hostName = entry.getKey(); + if (entry.getValue() != null) { + nmResource = entry.getValue(); + } + nm.init(hostName, nmResource, random.nextInt(heartbeatInterval), heartbeatInterval, rm, resourceUtilizationRatio); nmMap.put(nm.getNode().getNodeID(), nm); runner.schedule(nm); http://git-wip-us.apache.org/repos/asf/hadoop/blob/78860372/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/utils/SLSUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/utils/SLSUtils.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/utils/SLSUtils.java index e914fe7..f2129d0 100644 --- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/utils/SLSUtils.java +++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/utils/SLSUtils.java @@ -22,6 +22,8 @@ import java.io.FileInputStream; import java.io.IOException; import java.io.InputStreamReader; import java.io.Reader; + +import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; import java.util.List; @@ -39,7 +41,11 @@ import org.apache.hadoop.tools.rumen.JobTraceReader; import org.apache.hadoop.tools.rumen.LoggedJob; import org.apache.hadoop.tools.rumen.LoggedTask; import org.apache.hadoop.tools.rumen.LoggedTaskAttempt; +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.ResourceInformation; import org.apache.hadoop.yarn.sls.conf.SLSConfiguration; +import org.apache.hadoop.yarn.util.resource.ResourceUtils; +import org.apache.hadoop.yarn.util.resource.Resources; @Private @Unstable @@ -145,9 +151,9 @@ public class SLSUtils { /** * parse the input node file, return each host name */ - public static Set<String> parseNodesFromNodeFile(String nodeFile) - throws IOException { - Set<String> nodeSet = new HashSet<String>(); + public static Map<String, Resource> parseNodesFromNodeFile(String nodeFile, + Resource nmDefaultResource) throws IOException { + Map<String, Resource> nodeResourceMap = new HashMap<>(); JsonFactory jsonF = new JsonFactory(); ObjectMapper mapper = new ObjectMapper(); Reader input = @@ -160,13 +166,21 @@ public class SLSUtils { List tasks = (List) jsonE.get("nodes"); for (Object o : tasks) { Map jsonNode = (Map) o; - nodeSet.add(rack + "/" + jsonNode.get("node")); + Resource nodeResource = Resources.clone(nmDefaultResource); + ResourceInformation[] infors = ResourceUtils.getResourceTypesArray(); + for (ResourceInformation info : infors) { + if (jsonNode.get(info.getName()) != null) { + nodeResource.setResourceValue(info.getName(), + Integer.parseInt(jsonNode.get(info.getName()).toString())); + } + } + nodeResourceMap.put(rack + "/" + jsonNode.get("node"), nodeResource); } } } finally { input.close(); } - return nodeSet; + return nodeResourceMap; } public static Set<? extends String> generateNodes(int numNodes, http://git-wip-us.apache.org/repos/asf/hadoop/blob/78860372/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/utils/TestSLSUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/utils/TestSLSUtils.java b/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/utils/TestSLSUtils.java index 30964a1..5e586b13 100644 --- a/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/utils/TestSLSUtils.java +++ b/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/utils/TestSLSUtils.java @@ -18,10 +18,13 @@ package org.apache.hadoop.yarn.sls.utils; +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.util.resource.Resources; import org.junit.Assert; import org.junit.Test; import java.util.HashSet; +import java.util.Map; import java.util.Set; public class TestSLSUtils { @@ -40,6 +43,28 @@ public class TestSLSUtils { } @Test + public void testParseNodesFromNodeFile() throws Exception { + String nodeFile = "src/test/resources/nodes.json"; + Map<String, Resource> nodeResourceMap = SLSUtils.parseNodesFromNodeFile( + nodeFile, Resources.createResource(1024, 2)); + Assert.assertEquals(20, nodeResourceMap.size()); + + nodeFile = "src/test/resources/nodes-with-resources.json"; + nodeResourceMap = SLSUtils.parseNodesFromNodeFile( + nodeFile, Resources.createResource(1024, 2)); + Assert.assertEquals(4, + nodeResourceMap.size()); + Assert.assertEquals(2048, + nodeResourceMap.get("/rack1/node1").getMemorySize()); + Assert.assertEquals(6, + nodeResourceMap.get("/rack1/node1").getVirtualCores()); + Assert.assertEquals(1024, + nodeResourceMap.get("/rack1/node2").getMemorySize()); + Assert.assertEquals(2, + nodeResourceMap.get("/rack1/node2").getVirtualCores()); + } + + @Test public void testGenerateNodes() { Set<? extends String> nodes = SLSUtils.generateNodes(3, 3); Assert.assertEquals("Number of nodes is wrong.", 3, nodes.size()); http://git-wip-us.apache.org/repos/asf/hadoop/blob/78860372/hadoop-tools/hadoop-sls/src/test/resources/nodes-with-resources.json ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-sls/src/test/resources/nodes-with-resources.json b/hadoop-tools/hadoop-sls/src/test/resources/nodes-with-resources.json new file mode 100644 index 0000000..0039181 --- /dev/null +++ b/hadoop-tools/hadoop-sls/src/test/resources/nodes-with-resources.json @@ -0,0 +1,19 @@ +{ + "rack": "rack1", + "nodes": [ + { + "node": "node1", + "memory-mb" : 2048, + "vcores" : 6 + }, + { + "node": "node2" + }, + { + "node": "node3" + }, + { + "node": "node4" + } + ] +} --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
