Author: acmurthy
Date: Thu Oct 3 21:55:22 2013
New Revision: 1529019
URL: http://svn.apache.org/r1529019
Log:
Merge -c 1529015 from trunk to branch-2.1-beta to fix YARN-890. Ensure
CapacityScheduler doesn't round-up metric for available resources. Contributed
by Xuan Gong & Hitesh Shah.
Modified:
hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/CHANGES.txt
hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java
hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
Modified: hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/CHANGES.txt?rev=1529019&r1=1529018&r2=1529019&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/CHANGES.txt
(original)
+++ hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/CHANGES.txt Thu
Oct 3 21:55:22 2013
@@ -76,6 +76,9 @@ Release 2.1.2 - UNRELEASED
YARN-876. Node resource is added twice when node comes back from unhealthy
to healthy. (Peng Zhang via Sandy Ryza)
+ YARN-890. Ensure CapacityScheduler doesn't round-up metric for available
+ resources. (Xuan Gong & Hitesh Shah via acmurthy)
+
Release 2.1.1-beta - 2013-09-23
INCOMPATIBLE CHANGES
Modified:
hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java?rev=1529019&r1=1529018&r2=1529019&view=diff
==============================================================================
---
hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java
(original)
+++
hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java
Thu Oct 3 21:55:22 2013
@@ -99,15 +99,11 @@ class CSQueueUtils {
Resources.divide(calculator, clusterResource,
usedResources, queueLimit);
}
-
+
childQueue.setUsedCapacity(usedCapacity);
childQueue.setAbsoluteUsedCapacity(absoluteUsedCapacity);
- Resource available =
- Resources.roundUp(
- calculator,
- Resources.subtract(queueLimit, usedResources),
- minimumAllocation);
+ Resource available = Resources.subtract(queueLimit, usedResources);
childQueue.getMetrics().setAvailableResourcesToQueue(
Resources.max(
calculator,
Modified:
hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java?rev=1529019&r1=1529018&r2=1529019&view=diff
==============================================================================
---
hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
(original)
+++
hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
Thu Oct 3 21:55:22 2013
@@ -283,8 +283,9 @@ public class TestLeafQueue {
// Setup some nodes
String host_0 = "127.0.0.1";
- FiCaSchedulerNode node_0 = TestUtils.getMockNode(host_0, DEFAULT_RACK, 0,
8*GB);
-
+ FiCaSchedulerNode node_0 = TestUtils.getMockNode(host_0, DEFAULT_RACK, 0,
+ 8*GB);
+
final int numNodes = 1;
Resource clusterResource =
Resources.createResource(numNodes * (8*GB), numNodes * 16);
@@ -300,7 +301,9 @@ public class TestLeafQueue {
// Only 1 container
a.assignContainers(clusterResource, node_0);
- assertEquals(6*GB, a.getMetrics().getAvailableMB());
+ assertEquals(
+ (int)(node_0.getTotalResource().getMemory() * a.getCapacity()) -
(1*GB),
+ a.getMetrics().getAvailableMB());
}
@Test
@@ -405,8 +408,9 @@ public class TestLeafQueue {
// Setup some nodes
String host_0 = "127.0.0.1";
- FiCaSchedulerNode node_0 = TestUtils.getMockNode(host_0, DEFAULT_RACK, 0,
8*GB);
-
+ FiCaSchedulerNode node_0 = TestUtils.getMockNode(host_0, DEFAULT_RACK, 0,
+ 8*GB);
+
final int numNodes = 1;
Resource clusterResource =
Resources.createResource(numNodes * (8*GB), numNodes * 16);
@@ -493,12 +497,14 @@ public class TestLeafQueue {
a.completedContainer(clusterResource, app_1, node_0, rmContainer,
null, RMContainerEventType.KILL, null);
}
+
assertEquals(0*GB, a.getUsedResources().getMemory());
assertEquals(0*GB, app_0.getCurrentConsumption().getMemory());
assertEquals(0*GB, app_1.getCurrentConsumption().getMemory());
assertEquals(0*GB, a.getMetrics().getReservedMB());
assertEquals(0*GB, a.getMetrics().getAllocatedMB());
- assertEquals(1*GB, a.getMetrics().getAvailableMB());
+ assertEquals((int)(a.getCapacity() *
node_0.getTotalResource().getMemory()),
+ a.getMetrics().getAvailableMB());
}
@Test
