[2/4] hadoop git commit: YARN-8138. Add unit test to validate queue priority preemption works under node partition. (Zian Chen via wangda)

2018-04-16 Thread xyao 
YARN-8138. Add unit test to validate queue priority preemption works under node 
partition. (Zian Chen via wangda)

Change-Id: Ibebfab98a714c12c2dc643b6d7b9754a7f813632
(cherry picked from commit 6ee62e6b1c9b4bc3447ce870446068e626b1a492)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/896b473f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/896b473f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/896b473f

Branch: refs/heads/HDFS-7240
Commit: 896b473f1b477976e449184ceea075bedd71d6e8
Parents: 669eb7b
Author: Wangda Tan 
Authored: Sat Apr 14 11:04:49 2018 -0700
Committer: Wangda Tan 
Committed: Sat Apr 14 11:08:36 2018 -0700

--
 ...TestCapacitySchedulerSurgicalPreemption.java | 150 +++
 1 file changed, 150 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/896b473f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerSurgicalPreemption.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerSurgicalPreemption.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerSurgicalPreemption.java
index 9b183c0..2aff82d 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerSurgicalPreemption.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerSurgicalPreemption.java
@@ -18,8 +18,12 @@
 
 package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
 
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
 import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.NodeLabel;
 import org.apache.hadoop.yarn.api.records.Priority;
 import org.apache.hadoop.yarn.api.records.ResourceRequest;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
@@ -38,18 +42,24 @@ import 
org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
 import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp;
 import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent;
 import org.apache.hadoop.yarn.util.resource.Resources;
+import org.hamcrest.CoreMatchers;
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
 
+import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
 import java.util.Set;
 
+import static 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.PREFIX;
+import static org.hamcrest.MatcherAssert.assertThat;
+
 public class TestCapacitySchedulerSurgicalPreemption
 extends CapacitySchedulerPreemptionTestBase {
 
+  private static final int NUM_NM = 5;
   @Override
   @Before
   public void setUp() throws Exception {
@@ -860,6 +870,146 @@ public class TestCapacitySchedulerSurgicalPreemption
 rm1.close();
   }
 
+  private void initializeConfProperties(CapacitySchedulerConfiguration conf)
+  throws IOException {
+
+conf.setQueues("root", new String[] {"A", "B"});
+conf.setCapacity("root.A", 50);
+conf.setCapacity("root.B", 50);
+conf.setQueuePriority("root.A", 1);
+conf.setQueuePriority("root.B", 2);
+
+conf.set(PREFIX + "root.ordering-policy", "priority-utilization");
+conf.set(PREFIX + 
"ordering-policy.priority-utilization.underutilized-preemption.enabled", 
"true");
+conf.set(PREFIX + 
"ordering-policy.priority-utilization.underutilized-preemption.allow-move-reservation",
 "false");
+conf.set(PREFIX + 
"ordering-policy.priority-utilization.underutilized-preemption.reserved-container-delay-ms",
 "0");
+conf.set(PREFIX + "root.accessible-node-labels.x.capacity", "100");
+
+// Setup queue access to node labels
+conf.set(PREFIX + "root.A.accessible-node-labels", "x");
+conf.set(PREFIX + "root.B.accessible-node-labels", "x");
+conf.set(PREFIX + "root.A.default-node-label-expression", "x");
+conf.set(PREFIX + "root.B.default-node-label-expression"

hadoop git commit: YARN-8138. Add unit test to validate queue priority preemption works under node partition. (Zian Chen via wangda)

2018-04-14 Thread wangda 
Repository: hadoop
Updated Branches:
  refs/heads/trunk 669eb7bde -> 896b473f1


YARN-8138. Add unit test to validate queue priority preemption works under node 
partition. (Zian Chen via wangda)

Change-Id: Ibebfab98a714c12c2dc643b6d7b9754a7f813632
(cherry picked from commit 6ee62e6b1c9b4bc3447ce870446068e626b1a492)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/896b473f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/896b473f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/896b473f

Branch: refs/heads/trunk
Commit: 896b473f1b477976e449184ceea075bedd71d6e8
Parents: 669eb7b
Author: Wangda Tan 
Authored: Sat Apr 14 11:04:49 2018 -0700
Committer: Wangda Tan 
Committed: Sat Apr 14 11:08:36 2018 -0700

--
 ...TestCapacitySchedulerSurgicalPreemption.java | 150 +++
 1 file changed, 150 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/896b473f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerSurgicalPreemption.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerSurgicalPreemption.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerSurgicalPreemption.java
index 9b183c0..2aff82d 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerSurgicalPreemption.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerSurgicalPreemption.java
@@ -18,8 +18,12 @@
 
 package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
 
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
 import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.NodeLabel;
 import org.apache.hadoop.yarn.api.records.Priority;
 import org.apache.hadoop.yarn.api.records.ResourceRequest;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
@@ -38,18 +42,24 @@ import 
org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
 import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp;
 import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent;
 import org.apache.hadoop.yarn.util.resource.Resources;
+import org.hamcrest.CoreMatchers;
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
 
+import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
 import java.util.Set;
 
+import static 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.PREFIX;
+import static org.hamcrest.MatcherAssert.assertThat;
+
 public class TestCapacitySchedulerSurgicalPreemption
 extends CapacitySchedulerPreemptionTestBase {
 
+  private static final int NUM_NM = 5;
   @Override
   @Before
   public void setUp() throws Exception {
@@ -860,6 +870,146 @@ public class TestCapacitySchedulerSurgicalPreemption
 rm1.close();
   }
 
+  private void initializeConfProperties(CapacitySchedulerConfiguration conf)
+  throws IOException {
+
+conf.setQueues("root", new String[] {"A", "B"});
+conf.setCapacity("root.A", 50);
+conf.setCapacity("root.B", 50);
+conf.setQueuePriority("root.A", 1);
+conf.setQueuePriority("root.B", 2);
+
+conf.set(PREFIX + "root.ordering-policy", "priority-utilization");
+conf.set(PREFIX + 
"ordering-policy.priority-utilization.underutilized-preemption.enabled", 
"true");
+conf.set(PREFIX + 
"ordering-policy.priority-utilization.underutilized-preemption.allow-move-reservation",
 "false");
+conf.set(PREFIX + 
"ordering-policy.priority-utilization.underutilized-preemption.reserved-container-delay-ms",
 "0");
+conf.set(PREFIX + "root.accessible-node-labels.x.capacity", "100");
+
+// Setup queue access to node labels
+conf.set(PREFIX + "root.A.accessible-node-labels", "x");
+conf.set(PREFIX + "root.B.accessible-node-labels", "x");
+conf.set(PREFIX + "root.A.default-node-label-exp

hadoop git commit: YARN-8138. Add unit test to validate queue priority preemption works under node partition. (Zian Chen via wangda)

2018-04-14 Thread wangda 
Repository: hadoop
Updated Branches:
  refs/heads/branch-3.1 0b8868362 -> 6ee62e6b1


YARN-8138. Add unit test to validate queue priority preemption works under node 
partition. (Zian Chen via wangda)

Change-Id: Ibebfab98a714c12c2dc643b6d7b9754a7f813632


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6ee62e6b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6ee62e6b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6ee62e6b

Branch: refs/heads/branch-3.1
Commit: 6ee62e6b1c9b4bc3447ce870446068e626b1a492
Parents: 0b88683
Author: Wangda Tan 
Authored: Sat Apr 14 11:04:49 2018 -0700
Committer: Wangda Tan 
Committed: Sat Apr 14 11:04:49 2018 -0700

--
 ...TestCapacitySchedulerSurgicalPreemption.java | 150 +++
 1 file changed, 150 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/6ee62e6b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerSurgicalPreemption.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerSurgicalPreemption.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerSurgicalPreemption.java
index 9b183c0..2aff82d 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerSurgicalPreemption.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerSurgicalPreemption.java
@@ -18,8 +18,12 @@
 
 package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
 
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
 import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.NodeLabel;
 import org.apache.hadoop.yarn.api.records.Priority;
 import org.apache.hadoop.yarn.api.records.ResourceRequest;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
@@ -38,18 +42,24 @@ import 
org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
 import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp;
 import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent;
 import org.apache.hadoop.yarn.util.resource.Resources;
+import org.hamcrest.CoreMatchers;
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
 
+import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
 import java.util.Set;
 
+import static 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.PREFIX;
+import static org.hamcrest.MatcherAssert.assertThat;
+
 public class TestCapacitySchedulerSurgicalPreemption
 extends CapacitySchedulerPreemptionTestBase {
 
+  private static final int NUM_NM = 5;
   @Override
   @Before
   public void setUp() throws Exception {
@@ -860,6 +870,146 @@ public class TestCapacitySchedulerSurgicalPreemption
 rm1.close();
   }
 
+  private void initializeConfProperties(CapacitySchedulerConfiguration conf)
+  throws IOException {
+
+conf.setQueues("root", new String[] {"A", "B"});
+conf.setCapacity("root.A", 50);
+conf.setCapacity("root.B", 50);
+conf.setQueuePriority("root.A", 1);
+conf.setQueuePriority("root.B", 2);
+
+conf.set(PREFIX + "root.ordering-policy", "priority-utilization");
+conf.set(PREFIX + 
"ordering-policy.priority-utilization.underutilized-preemption.enabled", 
"true");
+conf.set(PREFIX + 
"ordering-policy.priority-utilization.underutilized-preemption.allow-move-reservation",
 "false");
+conf.set(PREFIX + 
"ordering-policy.priority-utilization.underutilized-preemption.reserved-container-delay-ms",
 "0");
+conf.set(PREFIX + "root.accessible-node-labels.x.capacity", "100");
+
+// Setup queue access to node labels
+conf.set(PREFIX + "root.A.accessible-node-labels", "x");
+conf.set(PREFIX + "root.B.accessible-node-labels", "x");
+conf.set(PREFIX + "root.A.default-node-label-expression", "x");
+conf.set(PREFIX + "root.B.default-node