[41/50] [abbrv] hadoop git commit: YARN-1015. FS should watch node resource utilization and allocate opportunistic containers if appropriate.

2018-04-10 Thread haibochen 
http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef338d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
index 1227de2..ac925c5 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
@@ -55,14 +55,20 @@ import org.apache.hadoop.yarn.MockApps;
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerExitStatus;
 import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
+import org.apache.hadoop.yarn.api.records.ContainerState;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.ExecutionType;
 import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
 import org.apache.hadoop.yarn.api.records.NodeId;
 import org.apache.hadoop.yarn.api.records.NodeState;
 import org.apache.hadoop.yarn.api.records.QueueInfo;
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.apache.hadoop.yarn.api.records.ResourceUtilization;
 import 
org.apache.hadoop.yarn.api.records.impl.pb.ApplicationSubmissionContextPBImpl;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.event.AsyncDispatcher;
@@ -72,6 +78,8 @@ import org.apache.hadoop.yarn.event.EventHandler;
 import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
 import org.apache.hadoop.yarn.security.YarnAuthorizationProvider;
+import org.apache.hadoop.yarn.server.api.records.OverAllocationInfo;
+import org.apache.hadoop.yarn.server.api.records.ResourceThresholds;
 import org.apache.hadoop.yarn.server.resourcemanager.ApplicationMasterService;
 import org.apache.hadoop.yarn.server.resourcemanager.MockAM;
 import org.apache.hadoop.yarn.server.resourcemanager.MockNM;
@@ -93,6 +101,7 @@ import 
org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
 import 
org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEventType;
 import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
 import 
org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeResourceUpdateEvent;
+import 
org.apache.hadoop.yarn.server.resourcemanager.rmnode.UpdatedContainerInfo;
 import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler;
 
 
@@ -1056,15 +1065,15 @@ public class TestFairScheduler extends 
FairSchedulerTestBase {
 assertEquals(
 YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
 scheduler.getQueueManager().getQueue("queue1").
-getResourceUsage().getMemorySize());
+getGuaranteedResourceUsage().getMemorySize());
 
 NodeUpdateSchedulerEvent updateEvent2 = new 
NodeUpdateSchedulerEvent(node2);
 scheduler.handle(updateEvent2);
 
 assertEquals(1024, scheduler.getQueueManager().getQueue("queue1").
-  getResourceUsage().getMemorySize());
+getGuaranteedResourceUsage().getMemorySize());
 assertEquals(2, scheduler.getQueueManager().getQueue("queue1").
-  getResourceUsage().getVirtualCores());
+getGuaranteedResourceUsage().getVirtualCores());
 
 // verify metrics
 QueueMetrics queue1Metrics = scheduler.getQueueManager().getQueue("queue1")
@@ -1099,7 +1108,7 @@ public class TestFairScheduler extends 
FairSchedulerTestBase {
 
 // Make sure queue 1 is allocated app capacity
 assertEquals(1024, scheduler.getQueueManager().getQueue("queue1").
-getResourceUsage().getMemorySize());
+getGuaranteedResourceUsage().getMemorySize());
 
 // Now queue 2 requests likewise
 ApplicationAttemptId attId = createSchedulingRequest(1024, "queue2", 
"user1", 1);
@@ -1109,7 +1118,7 @@ public class TestFairScheduler extends 
FairSchedulerTestBase {
 
 // Make sure queue 2 is waiting

[41/50] [abbrv] hadoop git commit: YARN-1015. FS should watch node resource utilization and allocate opportunistic containers if appropriate.

2018-01-28 Thread haibochen 
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bdf3e661/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
index ef5de39..8756c7d 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
@@ -55,14 +55,20 @@ import org.apache.hadoop.yarn.MockApps;
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerExitStatus;
 import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
+import org.apache.hadoop.yarn.api.records.ContainerState;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.ExecutionType;
 import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
 import org.apache.hadoop.yarn.api.records.NodeId;
 import org.apache.hadoop.yarn.api.records.NodeState;
 import org.apache.hadoop.yarn.api.records.QueueInfo;
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.apache.hadoop.yarn.api.records.ResourceUtilization;
 import 
org.apache.hadoop.yarn.api.records.impl.pb.ApplicationSubmissionContextPBImpl;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.event.AsyncDispatcher;
@@ -72,6 +78,8 @@ import org.apache.hadoop.yarn.event.EventHandler;
 import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
 import org.apache.hadoop.yarn.security.YarnAuthorizationProvider;
+import org.apache.hadoop.yarn.server.api.records.OverAllocationInfo;
+import org.apache.hadoop.yarn.server.api.records.ResourceThresholds;
 import org.apache.hadoop.yarn.server.resourcemanager.ApplicationMasterService;
 import org.apache.hadoop.yarn.server.resourcemanager.MockAM;
 import org.apache.hadoop.yarn.server.resourcemanager.MockNM;
@@ -93,6 +101,7 @@ import 
org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
 import 
org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEventType;
 import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
 import 
org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeResourceUpdateEvent;
+import 
org.apache.hadoop.yarn.server.resourcemanager.rmnode.UpdatedContainerInfo;
 import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler;
 
 
@@ -1056,15 +1065,15 @@ public class TestFairScheduler extends 
FairSchedulerTestBase {
 assertEquals(
 YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
 scheduler.getQueueManager().getQueue("queue1").
-getResourceUsage().getMemorySize());
+getGuaranteedResourceUsage().getMemorySize());
 
 NodeUpdateSchedulerEvent updateEvent2 = new 
NodeUpdateSchedulerEvent(node2);
 scheduler.handle(updateEvent2);
 
 assertEquals(1024, scheduler.getQueueManager().getQueue("queue1").
-  getResourceUsage().getMemorySize());
+getGuaranteedResourceUsage().getMemorySize());
 assertEquals(2, scheduler.getQueueManager().getQueue("queue1").
-  getResourceUsage().getVirtualCores());
+getGuaranteedResourceUsage().getVirtualCores());
 
 // verify metrics
 QueueMetrics queue1Metrics = scheduler.getQueueManager().getQueue("queue1")
@@ -1099,7 +1108,7 @@ public class TestFairScheduler extends 
FairSchedulerTestBase {
 
 // Make sure queue 1 is allocated app capacity
 assertEquals(1024, scheduler.getQueueManager().getQueue("queue1").
-getResourceUsage().getMemorySize());
+getGuaranteedResourceUsage().getMemorySize());
 
 // Now queue 2 requests likewise
 ApplicationAttemptId attId = createSchedulingRequest(1024, "queue2", 
"user1", 1);
@@ -1109,7 +1118,7 @@ public class TestFairScheduler extends 
FairSchedulerTestBase {
 
 // Make sure queue 2 is waiting