[12/50] [abbrv] hadoop git commit: YARN-7413. Support resource type in SLS (Contributed by Yufei Gu via Daniel Templeton)

2017-11-15 Thread aengineer 
YARN-7413. Support resource type in SLS (Contributed by Yufei Gu via Daniel 
Templeton)

Change-Id: Ic0a897c123c5d2f57aae757ca6bcf1dad7b90d2b


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ba813661
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ba813661
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ba813661

Branch: refs/heads/HDFS-7240
Commit: ba8136615ab66c450884614557eddc6509d63b7c
Parents: 462f6c4
Author: Daniel Templeton 
Authored: Thu Nov 9 12:09:48 2017 -0800
Committer: Daniel Templeton 
Committed: Thu Nov 9 12:09:48 2017 -0800

--
 .../org/apache/hadoop/yarn/sls/SLSRunner.java   | 86 +---
 .../hadoop/yarn/sls/conf/SLSConfiguration.java  |  6 +-
 .../yarn/sls/nodemanager/NMSimulator.java   | 15 ++--
 .../src/site/markdown/SchedulerLoadSimulator.md | 48 +++
 .../yarn/sls/nodemanager/TestNMSimulator.java   |  3 +-
 5 files changed, 117 insertions(+), 41 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba813661/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java
--
diff --git 
a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java
 
b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java
index 9d6c3aa..ad4310f 100644
--- 
a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java
+++ 
b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java
@@ -62,6 +62,7 @@ import org.apache.hadoop.yarn.api.records.NodeId;
 import org.apache.hadoop.yarn.api.records.NodeState;
 import org.apache.hadoop.yarn.api.records.ReservationId;
 import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceInformation;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
@@ -84,6 +85,7 @@ import org.apache.hadoop.yarn.sls.synthetic.SynthJob;
 import org.apache.hadoop.yarn.sls.synthetic.SynthTraceJobProducer;
 import org.apache.hadoop.yarn.sls.utils.SLSUtils;
 import org.apache.hadoop.yarn.util.UTCClock;
+import org.apache.hadoop.yarn.util.resource.ResourceUtils;
 import org.apache.hadoop.yarn.util.resource.Resources;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -99,7 +101,7 @@ public class SLSRunner extends Configured implements Tool {
 
   // NM simulator
   private HashMap nmMap;
-  private int nmMemoryMB, nmVCores;
+  private Resource nodeManagerResource;
   private String nodeFile;
 
   // AM simulator
@@ -178,6 +180,30 @@ public class SLSRunner extends Configured implements Tool {
 amClassMap.put(amType, Class.forName(tempConf.get(key)));
   }
 }
+
+nodeManagerResource = getNodeManagerResource();
+  }
+
+  private Resource getNodeManagerResource() {
+Resource resource = Resources.createResource(0);
+ResourceInformation[] infors = ResourceUtils.getResourceTypesArray();
+for (ResourceInformation info : infors) {
+  long value;
+  if (info.getName().equals(ResourceInformation.MEMORY_URI)) {
+value = getConf().getInt(SLSConfiguration.NM_MEMORY_MB,
+SLSConfiguration.NM_MEMORY_MB_DEFAULT);
+  } else if (info.getName().equals(ResourceInformation.VCORES_URI)) {
+value = getConf().getInt(SLSConfiguration.NM_VCORES,
+SLSConfiguration.NM_VCORES_DEFAULT);
+  } else {
+value = getConf().getLong(SLSConfiguration.NM_PREFIX +
+info.getName(), SLSConfiguration.NM_RESOURCE_DEFAULT);
+  }
+
+  resource.setResourceValue(info.getName(), value);
+}
+
+return resource;
   }
 
   /**
@@ -261,10 +287,6 @@ public class SLSRunner extends Configured implements Tool {
 
   private void startNM() throws YarnException, IOException {
 // nm configuration
-nmMemoryMB = getConf().getInt(SLSConfiguration.NM_MEMORY_MB,
-SLSConfiguration.NM_MEMORY_MB_DEFAULT);
-nmVCores = getConf().getInt(SLSConfiguration.NM_VCORES,
-SLSConfiguration.NM_VCORES_DEFAULT);
 int heartbeatInterval =
 getConf().getInt(SLSConfiguration.NM_HEARTBEAT_INTERVAL_MS,
 SLSConfiguration.NM_HEARTBEAT_INTERVAL_MS_DEFAULT);
@@ -304,7 +326,7 @@ public class SLSRunner extends Configured implements Tool {
 for (String hostName : nodeSet) {
   // we randomize the heartbeat start time from zero to 1 interval
   NMSimulator nm = new NMSimulator();
-  nm.init(hostName, nmMemoryMB, nmVCores, 
random.nextInt(heartbeatInterval),
+  nm.init(hostName, nodeManagerResource,

[21/50] [abbrv] hadoop git commit: YARN-7413. Support resource type in SLS (Contributed by Yufei Gu via Daniel Templeton)

2017-11-13 Thread kkaranasos 
YARN-7413. Support resource type in SLS (Contributed by Yufei Gu via Daniel 
Templeton)

Change-Id: Ic0a897c123c5d2f57aae757ca6bcf1dad7b90d2b


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ba813661
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ba813661
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ba813661

Branch: refs/heads/YARN-6592
Commit: ba8136615ab66c450884614557eddc6509d63b7c
Parents: 462f6c4
Author: Daniel Templeton 
Authored: Thu Nov 9 12:09:48 2017 -0800
Committer: Daniel Templeton 
Committed: Thu Nov 9 12:09:48 2017 -0800

--
 .../org/apache/hadoop/yarn/sls/SLSRunner.java   | 86 +---
 .../hadoop/yarn/sls/conf/SLSConfiguration.java  |  6 +-
 .../yarn/sls/nodemanager/NMSimulator.java   | 15 ++--
 .../src/site/markdown/SchedulerLoadSimulator.md | 48 +++
 .../yarn/sls/nodemanager/TestNMSimulator.java   |  3 +-
 5 files changed, 117 insertions(+), 41 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba813661/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java
--
diff --git 
a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java
 
b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java
index 9d6c3aa..ad4310f 100644
--- 
a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java
+++ 
b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java
@@ -62,6 +62,7 @@ import org.apache.hadoop.yarn.api.records.NodeId;
 import org.apache.hadoop.yarn.api.records.NodeState;
 import org.apache.hadoop.yarn.api.records.ReservationId;
 import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceInformation;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
@@ -84,6 +85,7 @@ import org.apache.hadoop.yarn.sls.synthetic.SynthJob;
 import org.apache.hadoop.yarn.sls.synthetic.SynthTraceJobProducer;
 import org.apache.hadoop.yarn.sls.utils.SLSUtils;
 import org.apache.hadoop.yarn.util.UTCClock;
+import org.apache.hadoop.yarn.util.resource.ResourceUtils;
 import org.apache.hadoop.yarn.util.resource.Resources;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -99,7 +101,7 @@ public class SLSRunner extends Configured implements Tool {
 
   // NM simulator
   private HashMap nmMap;
-  private int nmMemoryMB, nmVCores;
+  private Resource nodeManagerResource;
   private String nodeFile;
 
   // AM simulator
@@ -178,6 +180,30 @@ public class SLSRunner extends Configured implements Tool {
 amClassMap.put(amType, Class.forName(tempConf.get(key)));
   }
 }
+
+nodeManagerResource = getNodeManagerResource();
+  }
+
+  private Resource getNodeManagerResource() {
+Resource resource = Resources.createResource(0);
+ResourceInformation[] infors = ResourceUtils.getResourceTypesArray();
+for (ResourceInformation info : infors) {
+  long value;
+  if (info.getName().equals(ResourceInformation.MEMORY_URI)) {
+value = getConf().getInt(SLSConfiguration.NM_MEMORY_MB,
+SLSConfiguration.NM_MEMORY_MB_DEFAULT);
+  } else if (info.getName().equals(ResourceInformation.VCORES_URI)) {
+value = getConf().getInt(SLSConfiguration.NM_VCORES,
+SLSConfiguration.NM_VCORES_DEFAULT);
+  } else {
+value = getConf().getLong(SLSConfiguration.NM_PREFIX +
+info.getName(), SLSConfiguration.NM_RESOURCE_DEFAULT);
+  }
+
+  resource.setResourceValue(info.getName(), value);
+}
+
+return resource;
   }
 
   /**
@@ -261,10 +287,6 @@ public class SLSRunner extends Configured implements Tool {
 
   private void startNM() throws YarnException, IOException {
 // nm configuration
-nmMemoryMB = getConf().getInt(SLSConfiguration.NM_MEMORY_MB,
-SLSConfiguration.NM_MEMORY_MB_DEFAULT);
-nmVCores = getConf().getInt(SLSConfiguration.NM_VCORES,
-SLSConfiguration.NM_VCORES_DEFAULT);
 int heartbeatInterval =
 getConf().getInt(SLSConfiguration.NM_HEARTBEAT_INTERVAL_MS,
 SLSConfiguration.NM_HEARTBEAT_INTERVAL_MS_DEFAULT);
@@ -304,7 +326,7 @@ public class SLSRunner extends Configured implements Tool {
 for (String hostName : nodeSet) {
   // we randomize the heartbeat start time from zero to 1 interval
   NMSimulator nm = new NMSimulator();
-  nm.init(hostName, nmMemoryMB, nmVCores, 
random.nextInt(heartbeatInterval),
+  nm.init(hostName, nodeManagerResource,

[32/50] [abbrv] hadoop git commit: YARN-7413. Support resource type in SLS (Contributed by Yufei Gu via Daniel Templeton)

2017-11-11 Thread sunilg 
YARN-7413. Support resource type in SLS (Contributed by Yufei Gu via Daniel 
Templeton)

Change-Id: Ic0a897c123c5d2f57aae757ca6bcf1dad7b90d2b


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ba813661
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ba813661
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ba813661

Branch: refs/heads/YARN-5881
Commit: ba8136615ab66c450884614557eddc6509d63b7c
Parents: 462f6c4
Author: Daniel Templeton 
Authored: Thu Nov 9 12:09:48 2017 -0800
Committer: Daniel Templeton 
Committed: Thu Nov 9 12:09:48 2017 -0800

--
 .../org/apache/hadoop/yarn/sls/SLSRunner.java   | 86 +---
 .../hadoop/yarn/sls/conf/SLSConfiguration.java  |  6 +-
 .../yarn/sls/nodemanager/NMSimulator.java   | 15 ++--
 .../src/site/markdown/SchedulerLoadSimulator.md | 48 +++
 .../yarn/sls/nodemanager/TestNMSimulator.java   |  3 +-
 5 files changed, 117 insertions(+), 41 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba813661/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java
--
diff --git 
a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java
 
b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java
index 9d6c3aa..ad4310f 100644
--- 
a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java
+++ 
b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java
@@ -62,6 +62,7 @@ import org.apache.hadoop.yarn.api.records.NodeId;
 import org.apache.hadoop.yarn.api.records.NodeState;
 import org.apache.hadoop.yarn.api.records.ReservationId;
 import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceInformation;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
@@ -84,6 +85,7 @@ import org.apache.hadoop.yarn.sls.synthetic.SynthJob;
 import org.apache.hadoop.yarn.sls.synthetic.SynthTraceJobProducer;
 import org.apache.hadoop.yarn.sls.utils.SLSUtils;
 import org.apache.hadoop.yarn.util.UTCClock;
+import org.apache.hadoop.yarn.util.resource.ResourceUtils;
 import org.apache.hadoop.yarn.util.resource.Resources;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -99,7 +101,7 @@ public class SLSRunner extends Configured implements Tool {
 
   // NM simulator
   private HashMap nmMap;
-  private int nmMemoryMB, nmVCores;
+  private Resource nodeManagerResource;
   private String nodeFile;
 
   // AM simulator
@@ -178,6 +180,30 @@ public class SLSRunner extends Configured implements Tool {
 amClassMap.put(amType, Class.forName(tempConf.get(key)));
   }
 }
+
+nodeManagerResource = getNodeManagerResource();
+  }
+
+  private Resource getNodeManagerResource() {
+Resource resource = Resources.createResource(0);
+ResourceInformation[] infors = ResourceUtils.getResourceTypesArray();
+for (ResourceInformation info : infors) {
+  long value;
+  if (info.getName().equals(ResourceInformation.MEMORY_URI)) {
+value = getConf().getInt(SLSConfiguration.NM_MEMORY_MB,
+SLSConfiguration.NM_MEMORY_MB_DEFAULT);
+  } else if (info.getName().equals(ResourceInformation.VCORES_URI)) {
+value = getConf().getInt(SLSConfiguration.NM_VCORES,
+SLSConfiguration.NM_VCORES_DEFAULT);
+  } else {
+value = getConf().getLong(SLSConfiguration.NM_PREFIX +
+info.getName(), SLSConfiguration.NM_RESOURCE_DEFAULT);
+  }
+
+  resource.setResourceValue(info.getName(), value);
+}
+
+return resource;
   }
 
   /**
@@ -261,10 +287,6 @@ public class SLSRunner extends Configured implements Tool {
 
   private void startNM() throws YarnException, IOException {
 // nm configuration
-nmMemoryMB = getConf().getInt(SLSConfiguration.NM_MEMORY_MB,
-SLSConfiguration.NM_MEMORY_MB_DEFAULT);
-nmVCores = getConf().getInt(SLSConfiguration.NM_VCORES,
-SLSConfiguration.NM_VCORES_DEFAULT);
 int heartbeatInterval =
 getConf().getInt(SLSConfiguration.NM_HEARTBEAT_INTERVAL_MS,
 SLSConfiguration.NM_HEARTBEAT_INTERVAL_MS_DEFAULT);
@@ -304,7 +326,7 @@ public class SLSRunner extends Configured implements Tool {
 for (String hostName : nodeSet) {
   // we randomize the heartbeat start time from zero to 1 interval
   NMSimulator nm = new NMSimulator();
-  nm.init(hostName, nmMemoryMB, nmVCores, 
random.nextInt(heartbeatInterval),
+  nm.init(hostName, nodeManagerResource,

[2/2] hadoop git commit: YARN-7413. Support resource type in SLS (Contributed by Yufei Gu via Daniel Templeton)

2017-11-09 Thread yufei 
YARN-7413. Support resource type in SLS (Contributed by Yufei Gu via Daniel 
Templeton)

Change-Id: Ic0a897c123c5d2f57aae757ca6bcf1dad7b90d2b
(cherry picked from commit ba8136615ab66c450884614557eddc6509d63b7c)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/38f5d4cb
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/38f5d4cb
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/38f5d4cb

Branch: refs/heads/branch-3.0
Commit: 38f5d4cbba856495b4e3d6838abdb6b5f4233109
Parents: ddef7d0
Author: Daniel Templeton 
Authored: Thu Nov 9 12:09:48 2017 -0800
Committer: Yufei Gu 
Committed: Thu Nov 9 13:39:08 2017 -0800

--
 .../org/apache/hadoop/yarn/sls/SLSRunner.java   | 86 +---
 .../hadoop/yarn/sls/conf/SLSConfiguration.java  |  6 +-
 .../yarn/sls/nodemanager/NMSimulator.java   | 15 ++--
 .../src/site/markdown/SchedulerLoadSimulator.md | 48 +++
 .../yarn/sls/nodemanager/TestNMSimulator.java   |  3 +-
 5 files changed, 117 insertions(+), 41 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/38f5d4cb/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java
--
diff --git 
a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java
 
b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java
index 9d6c3aa..ad4310f 100644
--- 
a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java
+++ 
b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java
@@ -62,6 +62,7 @@ import org.apache.hadoop.yarn.api.records.NodeId;
 import org.apache.hadoop.yarn.api.records.NodeState;
 import org.apache.hadoop.yarn.api.records.ReservationId;
 import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceInformation;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
@@ -84,6 +85,7 @@ import org.apache.hadoop.yarn.sls.synthetic.SynthJob;
 import org.apache.hadoop.yarn.sls.synthetic.SynthTraceJobProducer;
 import org.apache.hadoop.yarn.sls.utils.SLSUtils;
 import org.apache.hadoop.yarn.util.UTCClock;
+import org.apache.hadoop.yarn.util.resource.ResourceUtils;
 import org.apache.hadoop.yarn.util.resource.Resources;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -99,7 +101,7 @@ public class SLSRunner extends Configured implements Tool {
 
   // NM simulator
   private HashMap nmMap;
-  private int nmMemoryMB, nmVCores;
+  private Resource nodeManagerResource;
   private String nodeFile;
 
   // AM simulator
@@ -178,6 +180,30 @@ public class SLSRunner extends Configured implements Tool {
 amClassMap.put(amType, Class.forName(tempConf.get(key)));
   }
 }
+
+nodeManagerResource = getNodeManagerResource();
+  }
+
+  private Resource getNodeManagerResource() {
+Resource resource = Resources.createResource(0);
+ResourceInformation[] infors = ResourceUtils.getResourceTypesArray();
+for (ResourceInformation info : infors) {
+  long value;
+  if (info.getName().equals(ResourceInformation.MEMORY_URI)) {
+value = getConf().getInt(SLSConfiguration.NM_MEMORY_MB,
+SLSConfiguration.NM_MEMORY_MB_DEFAULT);
+  } else if (info.getName().equals(ResourceInformation.VCORES_URI)) {
+value = getConf().getInt(SLSConfiguration.NM_VCORES,
+SLSConfiguration.NM_VCORES_DEFAULT);
+  } else {
+value = getConf().getLong(SLSConfiguration.NM_PREFIX +
+info.getName(), SLSConfiguration.NM_RESOURCE_DEFAULT);
+  }
+
+  resource.setResourceValue(info.getName(), value);
+}
+
+return resource;
   }
 
   /**
@@ -261,10 +287,6 @@ public class SLSRunner extends Configured implements Tool {
 
   private void startNM() throws YarnException, IOException {
 // nm configuration
-nmMemoryMB = getConf().getInt(SLSConfiguration.NM_MEMORY_MB,
-SLSConfiguration.NM_MEMORY_MB_DEFAULT);
-nmVCores = getConf().getInt(SLSConfiguration.NM_VCORES,
-SLSConfiguration.NM_VCORES_DEFAULT);
 int heartbeatInterval =
 getConf().getInt(SLSConfiguration.NM_HEARTBEAT_INTERVAL_MS,
 SLSConfiguration.NM_HEARTBEAT_INTERVAL_MS_DEFAULT);
@@ -304,7 +326,7 @@ public class SLSRunner extends Configured implements Tool {
 for (String hostName : nodeSet) {
   // we randomize the heartbeat start time from zero to 1 interval
   NMSimulator nm = new NMSimulator();
-  nm.init(hostName, nmMemoryMB, nmVCores, 
random.nextInt(heartbeatInterval),
+

hadoop git commit: YARN-7413. Support resource type in SLS (Contributed by Yufei Gu via Daniel Templeton)

2017-11-09 Thread templedf 
Repository: hadoop
Updated Branches:
  refs/heads/trunk 462f6c490 -> ba8136615


YARN-7413. Support resource type in SLS (Contributed by Yufei Gu via Daniel 
Templeton)

Change-Id: Ic0a897c123c5d2f57aae757ca6bcf1dad7b90d2b


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ba813661
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ba813661
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ba813661

Branch: refs/heads/trunk
Commit: ba8136615ab66c450884614557eddc6509d63b7c
Parents: 462f6c4
Author: Daniel Templeton 
Authored: Thu Nov 9 12:09:48 2017 -0800
Committer: Daniel Templeton 
Committed: Thu Nov 9 12:09:48 2017 -0800

--
 .../org/apache/hadoop/yarn/sls/SLSRunner.java   | 86 +---
 .../hadoop/yarn/sls/conf/SLSConfiguration.java  |  6 +-
 .../yarn/sls/nodemanager/NMSimulator.java   | 15 ++--
 .../src/site/markdown/SchedulerLoadSimulator.md | 48 +++
 .../yarn/sls/nodemanager/TestNMSimulator.java   |  3 +-
 5 files changed, 117 insertions(+), 41 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba813661/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java
--
diff --git 
a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java
 
b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java
index 9d6c3aa..ad4310f 100644
--- 
a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java
+++ 
b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java
@@ -62,6 +62,7 @@ import org.apache.hadoop.yarn.api.records.NodeId;
 import org.apache.hadoop.yarn.api.records.NodeState;
 import org.apache.hadoop.yarn.api.records.ReservationId;
 import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceInformation;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
@@ -84,6 +85,7 @@ import org.apache.hadoop.yarn.sls.synthetic.SynthJob;
 import org.apache.hadoop.yarn.sls.synthetic.SynthTraceJobProducer;
 import org.apache.hadoop.yarn.sls.utils.SLSUtils;
 import org.apache.hadoop.yarn.util.UTCClock;
+import org.apache.hadoop.yarn.util.resource.ResourceUtils;
 import org.apache.hadoop.yarn.util.resource.Resources;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -99,7 +101,7 @@ public class SLSRunner extends Configured implements Tool {
 
   // NM simulator
   private HashMap nmMap;
-  private int nmMemoryMB, nmVCores;
+  private Resource nodeManagerResource;
   private String nodeFile;
 
   // AM simulator
@@ -178,6 +180,30 @@ public class SLSRunner extends Configured implements Tool {
 amClassMap.put(amType, Class.forName(tempConf.get(key)));
   }
 }
+
+nodeManagerResource = getNodeManagerResource();
+  }
+
+  private Resource getNodeManagerResource() {
+Resource resource = Resources.createResource(0);
+ResourceInformation[] infors = ResourceUtils.getResourceTypesArray();
+for (ResourceInformation info : infors) {
+  long value;
+  if (info.getName().equals(ResourceInformation.MEMORY_URI)) {
+value = getConf().getInt(SLSConfiguration.NM_MEMORY_MB,
+SLSConfiguration.NM_MEMORY_MB_DEFAULT);
+  } else if (info.getName().equals(ResourceInformation.VCORES_URI)) {
+value = getConf().getInt(SLSConfiguration.NM_VCORES,
+SLSConfiguration.NM_VCORES_DEFAULT);
+  } else {
+value = getConf().getLong(SLSConfiguration.NM_PREFIX +
+info.getName(), SLSConfiguration.NM_RESOURCE_DEFAULT);
+  }
+
+  resource.setResourceValue(info.getName(), value);
+}
+
+return resource;
   }
 
   /**
@@ -261,10 +287,6 @@ public class SLSRunner extends Configured implements Tool {
 
   private void startNM() throws YarnException, IOException {
 // nm configuration
-nmMemoryMB = getConf().getInt(SLSConfiguration.NM_MEMORY_MB,
-SLSConfiguration.NM_MEMORY_MB_DEFAULT);
-nmVCores = getConf().getInt(SLSConfiguration.NM_VCORES,
-SLSConfiguration.NM_VCORES_DEFAULT);
 int heartbeatInterval =
 getConf().getInt(SLSConfiguration.NM_HEARTBEAT_INTERVAL_MS,
 SLSConfiguration.NM_HEARTBEAT_INTERVAL_MS_DEFAULT);
@@ -304,7 +326,7 @@ public class SLSRunner extends Configured implements Tool {
 for (String hostName : nodeSet) {
   // we randomize the heartbeat start time from zero to 1 interval
   NMSimulator nm = new NMSimulator();
-  nm.init(hostName, nmMemoryMB, nmVCores,