This is an automated email from the ASF dual-hosted git repository.
wwei pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.2 by this push:
new 6e2b091 YARN-9580. Fulfilled reservation information in assignment is
lost when transferring in ParentQueue#assignContainers. Contributed by Tao Yang.
6e2b091 is described below
commit 6e2b09151549c8474daf8a776f2aee5efe050ec2
Author: Weiwei Yang <[email protected]>
AuthorDate: Tue Jun 4 15:24:37 2019 +0800
YARN-9580. Fulfilled reservation information in assignment is lost when
transferring in ParentQueue#assignContainers. Contributed by Tao Yang.
---
.../scheduler/capacity/ParentQueue.java | 4 ++
.../capacity/TestCapacitySchedulerMultiNodes.java | 60 ++++++++++++++++++++++
2 files changed, 64 insertions(+)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
index 522c10e..0aefa69 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
@@ -627,6 +627,10 @@ public class ParentQueue extends AbstractCSQueue {
assignedToChild.getRequestLocalityType());
assignment.setExcessReservation(assignedToChild.getExcessReservation());
assignment.setContainersToKill(assignedToChild.getContainersToKill());
+ assignment.setFulfilledReservation(
+ assignedToChild.isFulfilledReservation());
+ assignment.setFulfilledReservedContainer(
+ assignedToChild.getFulfilledReservedContainer());
// Done if no child-queue assigned anything
if (Resources.greaterThan(resourceCalculator, clusterResource,
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerMultiNodes.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerMultiNodes.java
index c90af94..9218373 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerMultiNodes.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerMultiNodes.java
@@ -31,9 +31,12 @@ import org.apache.hadoop.yarn.server.resourcemanager.MockAM;
import org.apache.hadoop.yarn.server.resourcemanager.MockNM;
import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
+import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode;
import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNodeReport;
+import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp;
+import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent;
import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.MultiNodeSorter;
import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.MultiNodeSortingManager;
import org.apache.hadoop.yarn.util.resource.DominantResourceCalculator;
@@ -163,4 +166,61 @@ public class TestCapacitySchedulerMultiNodes extends
CapacitySchedulerTestBase {
}
rm.stop();
}
+
+ @Test(timeout=30000)
+ public void testAllocateForReservedContainer() throws Exception {
+ CapacitySchedulerConfiguration newConf =
+ new CapacitySchedulerConfiguration(conf);
+ newConf.set(YarnConfiguration.RM_PLACEMENT_CONSTRAINTS_HANDLER,
+ YarnConfiguration.SCHEDULER_RM_PLACEMENT_CONSTRAINTS_HANDLER);
+
newConf.setInt(CapacitySchedulerConfiguration.MULTI_NODE_SORTING_POLICY_NAME
+ + ".resource-based.sorting-interval.ms", 0);
+ newConf.setMaximumApplicationMasterResourcePerQueuePercent("root.default",
+ 1.0f);
+ MockRM rm1 = new MockRM(newConf);
+
+ rm1.start();
+ MockNM nm1 = rm1.registerNode("h1:1234", 8 * GB);
+ MockNM nm2 = rm1.registerNode("h2:1234", 8 * GB);
+
+ // launch an app to queue, AM container should be launched in nm1
+ RMApp app1 = rm1.submitApp(5 * GB, "app", "user", null, "default");
+ MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1);
+
+ // launch another app to queue, AM container should be launched in nm2
+ RMApp app2 = rm1.submitApp(5 * GB, "app", "user", null, "default");
+ MockAM am2 = MockRM.launchAndRegisterAM(app2, rm1, nm2);
+
+ CapacityScheduler cs = (CapacityScheduler) rm1.getResourceScheduler();
+ RMNode rmNode1 = rm1.getRMContext().getRMNodes().get(nm1.getNodeId());
+ FiCaSchedulerApp schedulerApp1 =
+ cs.getApplicationAttempt(am1.getApplicationAttemptId());
+ FiCaSchedulerApp schedulerApp2 =
+ cs.getApplicationAttempt(am2.getApplicationAttemptId());
+
+ /*
+ * Verify that reserved container will be allocated
+ * after node has sufficient resource.
+ */
+ // Ask a container with 6GB memory size for app2,
+ // nm1 will reserve a container for app2
+ am2.allocate("*", 6 * GB, 1, new ArrayList<>());
+ cs.handle(new NodeUpdateSchedulerEvent(rmNode1));
+
+ // Check containers of app1 and app2.
+ Assert.assertNotNull(cs.getNode(nm1.getNodeId()).getReservedContainer());
+ Assert.assertEquals(1, schedulerApp1.getLiveContainers().size());
+ Assert.assertEquals(1, schedulerApp2.getLiveContainers().size());
+ Assert.assertEquals(1, schedulerApp2.getReservedContainers().size());
+
+ // Kill app1 to release resource on nm1.
+ rm1.killApp(app1.getApplicationId());
+
+ // Trigger scheduling to allocate for reserved container on nm1.
+ cs.handle(new NodeUpdateSchedulerEvent(rmNode1));
+ Assert.assertNull(cs.getNode(nm1.getNodeId()).getReservedContainer());
+ Assert.assertEquals(2, schedulerApp2.getLiveContainers().size());
+
+ rm1.close();
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
