[13/44] hadoop git commit: YARN-8127. Resource leak when async scheduling is enabled. Contributed by Tao Yang.

2018-04-13 Thread xyao 
YARN-8127. Resource leak when async scheduling is enabled. Contributed by Tao 
Yang.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7eb783e2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7eb783e2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7eb783e2

Branch: refs/heads/HDFS-7240
Commit: 7eb783e2634d8c11fb646f1f2fdf597336325312
Parents: b0aff8a
Author: Weiwei Yang 
Authored: Wed Apr 11 17:15:25 2018 +0800
Committer: Weiwei Yang 
Committed: Wed Apr 11 17:15:25 2018 +0800

--
 .../scheduler/common/fica/FiCaSchedulerApp.java | 10 +++
 .../TestCapacitySchedulerAsyncScheduling.java   | 91 
 2 files changed, 101 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/7eb783e2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
index 32b2cad..3ec8191 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
@@ -339,6 +339,16 @@ public class FiCaSchedulerApp extends 
SchedulerApplicationAttempt {
 return false;
   }
 }
+// If allocate from reserved container, make sure node is still reserved
+if (allocation.getAllocateFromReservedContainer() != null
+&& reservedContainerOnNode == null) {
+  if (LOG.isDebugEnabled()) {
+LOG.debug("Try to allocate from reserved container " + allocation
+.getAllocateFromReservedContainer().getRmContainer()
+.getContainerId() + ", but node is not reserved");
+  }
+  return false;
+}
 
 // Do we have enough space on this node?
 Resource availableResource = Resources.clone(

http://git-wip-us.apache.org/repos/asf/hadoop/blob/7eb783e2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAsyncScheduling.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAsyncScheduling.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAsyncScheduling.java
index 18cd942..338b9f9 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAsyncScheduling.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAsyncScheduling.java
@@ -594,6 +594,97 @@ public class TestCapacitySchedulerAsyncScheduling {
 }
   }
 
+  // Testcase for YARN-8127
+  @Test (timeout = 3)
+  public void testCommitDuplicatedAllocateFromReservedProposals()
+  throws Exception {
+// disable async-scheduling for simulating complex scene
+Configuration disableAsyncConf = new Configuration(conf);
+disableAsyncConf.setBoolean(
+CapacitySchedulerConfiguration.SCHEDULE_ASYNCHRONOUSLY_ENABLE, false);
+
+// init RM & NMs
+final MockRM rm = new MockRM(disableAsyncConf);
+rm.start();
+final MockNM nm1 = rm.registerNode("192.168.0.1:1234", 8 * GB);
+rm.registerNode("192.168.0.2:2234", 8 * GB);
+
+// init scheduler & nodes
+while (
+((CapacityScheduler) rm.getRMContext().getScheduler()).getNodeTracker()
+.nodeCount() < 2) {
+  Thread.sleep(10);
+}
+Assert.assertEquals(2,
+((AbstractYarnScheduler) rm.getRMContext().getScheduler())
+.getNodeTracker().nodeCount());
+CapacityScheduler cs

hadoop git commit: YARN-8127. Resource leak when async scheduling is enabled. Contributed by Tao Yang.

2018-04-11 Thread wwei 
Repository: hadoop
Updated Branches:
  refs/heads/branch-3.1 86f543aa8 -> c2036af19


YARN-8127. Resource leak when async scheduling is enabled. Contributed by Tao 
Yang.

(cherry picked from commit 7eb783e2634d8c11fb646f1f2fdf597336325312)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c2036af1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c2036af1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c2036af1

Branch: refs/heads/branch-3.1
Commit: c2036af196f237539ecd4317ea510c697e8456ff
Parents: 86f543a
Author: Weiwei Yang 
Authored: Wed Apr 11 17:15:25 2018 +0800
Committer: Weiwei Yang 
Committed: Wed Apr 11 17:21:47 2018 +0800

--
 .../scheduler/common/fica/FiCaSchedulerApp.java | 10 +++
 .../TestCapacitySchedulerAsyncScheduling.java   | 91 
 2 files changed, 101 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/c2036af1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
index 32b2cad..3ec8191 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
@@ -339,6 +339,16 @@ public class FiCaSchedulerApp extends 
SchedulerApplicationAttempt {
 return false;
   }
 }
+// If allocate from reserved container, make sure node is still reserved
+if (allocation.getAllocateFromReservedContainer() != null
+&& reservedContainerOnNode == null) {
+  if (LOG.isDebugEnabled()) {
+LOG.debug("Try to allocate from reserved container " + allocation
+.getAllocateFromReservedContainer().getRmContainer()
+.getContainerId() + ", but node is not reserved");
+  }
+  return false;
+}
 
 // Do we have enough space on this node?
 Resource availableResource = Resources.clone(

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c2036af1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAsyncScheduling.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAsyncScheduling.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAsyncScheduling.java
index 18cd942..338b9f9 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAsyncScheduling.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAsyncScheduling.java
@@ -594,6 +594,97 @@ public class TestCapacitySchedulerAsyncScheduling {
 }
   }
 
+  // Testcase for YARN-8127
+  @Test (timeout = 3)
+  public void testCommitDuplicatedAllocateFromReservedProposals()
+  throws Exception {
+// disable async-scheduling for simulating complex scene
+Configuration disableAsyncConf = new Configuration(conf);
+disableAsyncConf.setBoolean(
+CapacitySchedulerConfiguration.SCHEDULE_ASYNCHRONOUSLY_ENABLE, false);
+
+// init RM & NMs
+final MockRM rm = new MockRM(disableAsyncConf);
+rm.start();
+final MockNM nm1 = rm.registerNode("192.168.0.1:1234", 8 * GB);
+rm.registerNode("192.168.0.2:2234", 8 * GB);
+
+// init scheduler & nodes
+while (
+((CapacityScheduler) rm.getRMContext().getScheduler()).getNodeTracker()
+.nodeCount() < 2) {
+  Thread.sleep(10);
+}
+Asser

hadoop git commit: YARN-8127. Resource leak when async scheduling is enabled. Contributed by Tao Yang.

2018-04-11 Thread wwei 
Repository: hadoop
Updated Branches:
  refs/heads/trunk b0aff8a96 -> 7eb783e26


YARN-8127. Resource leak when async scheduling is enabled. Contributed by Tao 
Yang.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7eb783e2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7eb783e2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7eb783e2

Branch: refs/heads/trunk
Commit: 7eb783e2634d8c11fb646f1f2fdf597336325312
Parents: b0aff8a
Author: Weiwei Yang 
Authored: Wed Apr 11 17:15:25 2018 +0800
Committer: Weiwei Yang 
Committed: Wed Apr 11 17:15:25 2018 +0800

--
 .../scheduler/common/fica/FiCaSchedulerApp.java | 10 +++
 .../TestCapacitySchedulerAsyncScheduling.java   | 91 
 2 files changed, 101 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/7eb783e2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
index 32b2cad..3ec8191 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
@@ -339,6 +339,16 @@ public class FiCaSchedulerApp extends 
SchedulerApplicationAttempt {
 return false;
   }
 }
+// If allocate from reserved container, make sure node is still reserved
+if (allocation.getAllocateFromReservedContainer() != null
+&& reservedContainerOnNode == null) {
+  if (LOG.isDebugEnabled()) {
+LOG.debug("Try to allocate from reserved container " + allocation
+.getAllocateFromReservedContainer().getRmContainer()
+.getContainerId() + ", but node is not reserved");
+  }
+  return false;
+}
 
 // Do we have enough space on this node?
 Resource availableResource = Resources.clone(

http://git-wip-us.apache.org/repos/asf/hadoop/blob/7eb783e2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAsyncScheduling.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAsyncScheduling.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAsyncScheduling.java
index 18cd942..338b9f9 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAsyncScheduling.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAsyncScheduling.java
@@ -594,6 +594,97 @@ public class TestCapacitySchedulerAsyncScheduling {
 }
   }
 
+  // Testcase for YARN-8127
+  @Test (timeout = 3)
+  public void testCommitDuplicatedAllocateFromReservedProposals()
+  throws Exception {
+// disable async-scheduling for simulating complex scene
+Configuration disableAsyncConf = new Configuration(conf);
+disableAsyncConf.setBoolean(
+CapacitySchedulerConfiguration.SCHEDULE_ASYNCHRONOUSLY_ENABLE, false);
+
+// init RM & NMs
+final MockRM rm = new MockRM(disableAsyncConf);
+rm.start();
+final MockNM nm1 = rm.registerNode("192.168.0.1:1234", 8 * GB);
+rm.registerNode("192.168.0.2:2234", 8 * GB);
+
+// init scheduler & nodes
+while (
+((CapacityScheduler) rm.getRMContext().getScheduler()).getNodeTracker()
+.nodeCount() < 2) {
+  Thread.sleep(10);
+}
+Assert.assertEquals(2,
+((AbstractYarnScheduler) rm.getRMContext().getSchedul