Author: sandy
Date: Mon Sep 30 22:03:06 2013
New Revision: 1527795
URL: http://svn.apache.org/r1527795
Log:
YARN-1221. With Fair Scheduler, reserved MB reported in RM web UI increases
indefinitely (Siqi Li via Sandy Ryza)
Modified:
hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AppSchedulable.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt?rev=1527795&r1=1527794&r2=1527795&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt Mon Sep 30
22:03:06 2013
@@ -83,6 +83,9 @@ Release 2.1.2 - UNRELEASED
YARN-1157. Fixed ResourceManager UI to behave correctly when apps like
distributed-shell do not set tracking urls. (Xuan Gong via vinodkv)
+ YARN-1221. With Fair Scheduler, reserved MB reported in RM web UI increases
+ indefinitely (Siqi Li via Sandy Ryza)
+
Release 2.1.1-beta - 2013-09-23
INCOMPATIBLE CHANGES
Modified:
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AppSchedulable.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AppSchedulable.java?rev=1527795&r1=1527794&r2=1527795&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AppSchedulable.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AppSchedulable.java
Mon Sep 30 22:03:06 2013
@@ -192,10 +192,6 @@ public class AppSchedulable extends Sche
RMContainer rmContainer = app.reserve(node, priority, null,
container);
node.reserveResource(app, priority, rmContainer);
- getMetrics().reserveResource(app.getUser(),
- container.getResource());
- scheduler.getRootQueueMetrics().reserveResource(app.getUser(),
- container.getResource());
}
else {
@@ -216,8 +212,6 @@ public class AppSchedulable extends Sche
node.unreserveResource(app);
getMetrics().unreserveResource(
app.getUser(), rmContainer.getContainer().getResource());
- scheduler.getRootQueueMetrics().unreserveResource(
- app.getUser(), rmContainer.getContainer().getResource());
}
/**
Modified:
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java?rev=1527795&r1=1527794&r2=1527795&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
Mon Sep 30 22:03:06 2013
@@ -449,6 +449,44 @@ public class TestFairScheduler {
Assert.assertEquals(3, queueManager.getLeafQueues().size());
}
+ @Test
+ public void testSchedulerRootQueueMetrics() throws InterruptedException {
+
+ // Add a node
+ RMNode node1 = MockNodes.newNodeInfo(1, Resources.createResource(1024));
+ NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1);
+ scheduler.handle(nodeEvent1);
+
+ // Queue 1 requests full capacity of node
+ createSchedulingRequest(1024, "queue1", "user1", 1);
+ scheduler.update();
+ NodeUpdateSchedulerEvent updateEvent = new NodeUpdateSchedulerEvent(node1);
+ scheduler.handle(updateEvent);
+
+ // Now queue 2 requests likewise
+ createSchedulingRequest(1024, "queue2", "user1", 1);
+ scheduler.update();
+ scheduler.handle(updateEvent);
+
+ // Make sure reserved memory gets updated correctly
+ assertEquals(1024, scheduler.rootMetrics.getReservedMB());
+
+ // Now another node checks in with capacity
+ RMNode node2 = MockNodes.newNodeInfo(1, Resources.createResource(1024));
+ NodeAddedSchedulerEvent nodeEvent2 = new NodeAddedSchedulerEvent(node2);
+ NodeUpdateSchedulerEvent updateEvent2 = new
NodeUpdateSchedulerEvent(node2);
+ scheduler.handle(nodeEvent2);
+ scheduler.handle(updateEvent2);
+
+
+ // The old reservation should still be there...
+ assertEquals(1024, scheduler.rootMetrics.getReservedMB());
+
+ // ... but it should disappear when we update the first node.
+ scheduler.handle(updateEvent);
+ assertEquals(0, scheduler.rootMetrics.getReservedMB());
+ }
+
@Test (timeout = 5000)
public void testSimpleContainerAllocation() {
// Add a node
