Author: jlowe
Date: Fri Apr 5 15:38:53 2013
New Revision: 1465013
URL: http://svn.apache.org/r1465013
Log:
YARN-525. make CS node-locality-delay refreshable. Contributed by Thomas Graves
Modified:
hadoop/common/branches/branch-0.23/hadoop-yarn-project/CHANGES.txt
hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
Modified: hadoop/common/branches/branch-0.23/hadoop-yarn-project/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-yarn-project/CHANGES.txt?rev=1465013&r1=1465012&r2=1465013&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-yarn-project/CHANGES.txt
(original)
+++ hadoop/common/branches/branch-0.23/hadoop-yarn-project/CHANGES.txt Fri Apr
5 15:38:53 2013
@@ -38,6 +38,8 @@ Release 0.23.7 - UNRELEASED
YARN-200. yarn log does not output all needed information, and is in a
binary format (Ravi Prakash via jlowe)
+ YARN-525. make CS node-locality-delay refreshable (Thomas Graves via jlowe)
+
OPTIMIZATIONS
YARN-357. App submission should not be synchronized (daryn)
Modified:
hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java?rev=1465013&r1=1465012&r2=1465013&view=diff
==============================================================================
---
hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
(original)
+++
hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
Fri Apr 5 15:38:53 2013
@@ -124,7 +124,7 @@ public class LeafQueue implements CSQueu
private final ActiveUsersManager activeUsersManager;
- private final int nodeLocalityDelay;
+ private int nodeLocalityDelay;
public LeafQueue(CapacitySchedulerContext cs,
String queueName, CSQueue parent,
@@ -189,9 +189,6 @@ public class LeafQueue implements CSQueu
Map<QueueACL, AccessControlList> acls =
cs.getConfiguration().getAcls(getQueuePath());
-
- this.nodeLocalityDelay =
- cs.getConfiguration().getNodeLocalityDelay();
setupQueueConfigs(
cs.getClusterResources(),
@@ -200,7 +197,7 @@ public class LeafQueue implements CSQueu
userLimit, userLimitFactor,
maxApplications, maxApplicationsPerUser,
maxActiveApplications, maxActiveApplicationsPerUser,
- state, acls);
+ state, acls, cs.getConfiguration().getNodeLocalityDelay());
if(LOG.isDebugEnabled()) {
LOG.debug("LeafQueue:" + " name=" + queueName
@@ -219,7 +216,8 @@ public class LeafQueue implements CSQueu
int userLimit, float userLimitFactor,
int maxApplications, int maxApplicationsPerUser,
int maxActiveApplications, int maxActiveApplicationsPerUser,
- QueueState state, Map<QueueACL, AccessControlList> acls)
+ QueueState state, Map<QueueACL, AccessControlList> acls,
+ int nodeLocalityDelay)
{
// Sanity check
CSQueueUtils.checkMaxCapacity(getQueueName(), capacity, maximumCapacity);
@@ -248,6 +246,8 @@ public class LeafQueue implements CSQueu
this.queueInfo.setCapacity(this.capacity);
this.queueInfo.setMaximumCapacity(this.maximumCapacity);
this.queueInfo.setQueueState(this.state);
+
+ this.nodeLocalityDelay = nodeLocalityDelay;
StringBuilder aclsString = new StringBuilder();
for (Map.Entry<QueueACL, AccessControlList> e : acls.entrySet()) {
@@ -310,7 +310,8 @@ public class LeafQueue implements CSQueu
"state = " + state +
" [= configuredState ]" + "\n" +
"acls = " + aclsString +
- " [= configuredAcls ]" + "\n");
+ " [= configuredAcls ]" + "\n" +
+ "nodeLocalityDelay = " + nodeLocalityDelay + "\n");
}
@Override
@@ -596,7 +597,8 @@ public class LeafQueue implements CSQueu
newlyParsedLeafQueue.getMaxApplicationsPerUser(),
newlyParsedLeafQueue.getMaximumActiveApplications(),
newlyParsedLeafQueue.getMaximumActiveApplicationsPerUser(),
- newlyParsedLeafQueue.state, newlyParsedLeafQueue.acls);
+ newlyParsedLeafQueue.state, newlyParsedLeafQueue.acls,
+ newlyParsedLeafQueue.getNodeLocalityDelay());
}
@Override
Modified:
hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java?rev=1465013&r1=1465012&r2=1465013&view=diff
==============================================================================
---
hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
(original)
+++
hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
Fri Apr 5 15:38:53 2013
@@ -129,6 +129,7 @@ public class TestLeafQueue {
private static final String C = "c";
private static final String C1 = "c1";
private static final String D = "d";
+ private static final String E = "e";
private void setupQueueConfiguration(
CapacitySchedulerConfiguration conf,
final String newRoot) {
@@ -140,7 +141,7 @@ public class TestLeafQueue {
conf.setAcl(CapacitySchedulerConfiguration.ROOT,
QueueACL.SUBMIT_APPLICATIONS, " ");
final String Q_newRoot = CapacitySchedulerConfiguration.ROOT + "." +
newRoot;
- conf.setQueues(Q_newRoot, new String[] {A, B, C, D});
+ conf.setQueues(Q_newRoot, new String[] {A, B, C, D, E});
conf.setCapacity(Q_newRoot, 100);
conf.setMaximumCapacity(Q_newRoot, 100);
conf.setAcl(Q_newRoot, QueueACL.SUBMIT_APPLICATIONS, " ");
@@ -166,9 +167,14 @@ public class TestLeafQueue {
conf.setCapacity(Q_C1, 100);
final String Q_D = Q_newRoot + "." + D;
- conf.setCapacity(Q_D, 10);
+ conf.setCapacity(Q_D, 9);
conf.setMaximumCapacity(Q_D, 11);
conf.setAcl(Q_D, QueueACL.SUBMIT_APPLICATIONS, "user_d");
+
+ final String Q_E = Q_newRoot + "." + E;
+ conf.setCapacity(Q_E, 1);
+ conf.setMaximumCapacity(Q_E, 1);
+ conf.setAcl(Q_E, QueueACL.SUBMIT_APPLICATIONS, "user_e");
}
@@ -1582,6 +1588,33 @@ public class TestLeafQueue {
c1.getQueueUserAclInfo(user), QueueACL.SUBMIT_APPLICATIONS));
}
+
+ @Test (timeout = 30000)
+ public void testNodeLocalityAfterQueueRefresh() throws Exception {
+
+ // Manipulate queue 'e'
+ LeafQueue e = stubLeafQueue((LeafQueue)queues.get(E));
+
+ // before reinitialization
+ assertEquals(0, e.getNodeLocalityDelay());
+
+ csConf.setInt(CapacitySchedulerConfiguration
+ .NODE_LOCALITY_DELAY, 60);
+ Map<String, CSQueue> newQueues = new HashMap<String, CSQueue>();
+ CSQueue newRoot =
+ CapacityScheduler.parseQueue(csContext, csConf, null,
+ CapacitySchedulerConfiguration.ROOT,
+ newQueues, queues,
+ CapacityScheduler.queueComparator,
+ CapacityScheduler.applicationComparator,
+ TestUtils.spyHook);
+ queues = newQueues;
+ root.reinitialize(newRoot, cs.getClusterResources());
+
+ // after reinitialization
+ assertEquals(60, e.getNodeLocalityDelay());
+ }
+
@After
public void tearDown() throws Exception {
