Author: sandy
Date: Fri May 23 22:38:52 2014
New Revision: 1597204
URL: http://svn.apache.org/r1597204
Log:
YARN-2012. Fair Scheduler: allow default queue placement rule to take an
arbitrary queue (Ashwin Shankar via Sandy Ryza)
Modified:
hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueuePlacementRule.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueuePlacementPolicy.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm
Modified: hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt?rev=1597204&r1=1597203&r2=1597204&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt Fri May 23 22:38:52 2014
@@ -99,6 +99,9 @@ Release 2.5.0 - UNRELEASED
YARN-1937. Added owner-only ACLs support for Timeline Client and server.
(Zhijie Shen via vinodkv)
+ YARN-2012. Fair Scheduler: allow default queue placement rule to take an
+ arbitrary queue (Ashwin Shankar via Sandy Ryza)
+
OPTIMIZATIONS
BUG FIXES
Modified:
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueuePlacementRule.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueuePlacementRule.java?rev=1597204&r1=1597203&r2=1597204&view=diff
==============================================================================
---
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueuePlacementRule.java
(original)
+++
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueuePlacementRule.java
Fri May 23 22:38:52 2014
@@ -276,21 +276,38 @@ public abstract class QueuePlacementRule
}
/**
- * Places all apps in the default queue
+ * Places apps in the specified default queue. If no default queue is
+ * specified the app is placed in root.default queue.
*/
public static class Default extends QueuePlacementRule {
+ private String defaultQueueName;
+
+ @Override
+ public void initializeFromXml(Element el)
+ throws AllocationConfigurationException {
+ defaultQueueName = el.getAttribute("queue");
+ if (defaultQueueName != null && !defaultQueueName.isEmpty()) {
+ if (!defaultQueueName.startsWith("root.")) {
+ defaultQueueName = "root." + defaultQueueName;
+ }
+ } else {
+ defaultQueueName = "root." + YarnConfiguration.DEFAULT_QUEUE_NAME;
+ }
+ super.initializeFromXml(el);
+ }
+
@Override
protected String getQueueForApp(String requestedQueue, String user,
Groups groups, Map<FSQueueType, Set<String>> configuredQueues) {
- return "root." + YarnConfiguration.DEFAULT_QUEUE_NAME;
+ return defaultQueueName;
}
-
+
@Override
public boolean isTerminal() {
return true;
}
}
-
+
/**
* Rejects all apps
*/
Modified:
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueuePlacementPolicy.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueuePlacementPolicy.java?rev=1597204&r1=1597203&r2=1597204&view=diff
==============================================================================
---
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueuePlacementPolicy.java
(original)
+++
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueuePlacementPolicy.java
Fri May 23 22:38:52 2014
@@ -134,6 +134,22 @@ public class TestQueuePlacementPolicy {
}
@Test
+ public void testDefaultRuleWithQueueAttribute() throws Exception {
+ // This test covers the use case where we would like default rule
+ // to point to a different queue by default rather than root.default
+ configuredQueues.get(FSQueueType.LEAF).add("root.someDefaultQueue");
+ StringBuffer sb = new StringBuffer();
+ sb.append("<queuePlacementPolicy>");
+ sb.append(" <rule name='specified' create='false' />");
+ sb.append(" <rule name='default' queue='root.someDefaultQueue'/>");
+ sb.append("</queuePlacementPolicy>");
+
+ QueuePlacementPolicy policy = parse(sb.toString());
+ assertEquals("root.someDefaultQueue",
+ policy.assignAppToQueue("root.default", "user1"));
+ }
+
+ @Test
public void testNestedUserQueueParsingErrors() {
// No nested rule specified in hierarchical user queue
StringBuffer sb = new StringBuffer();
@@ -311,6 +327,25 @@ public class TestQueuePlacementPolicy {
policy.assignAppToQueue("root.parent2", "user2"));
}
+ @Test
+ public void testNestedUserQueueDefaultRule() throws Exception {
+ // This test covers the use case where we would like user queues to be
+ // created under a default parent queue
+ configuredQueues.get(FSQueueType.PARENT).add("root.parentq");
+ StringBuffer sb = new StringBuffer();
+ sb.append("<queuePlacementPolicy>");
+ sb.append(" <rule name='specified' create='false' />");
+ sb.append(" <rule name='nestedUserQueue'>");
+ sb.append(" <rule name='default' queue='root.parentq'/>");
+ sb.append(" </rule>");
+ sb.append(" <rule name='default' />");
+ sb.append("</queuePlacementPolicy>");
+
+ QueuePlacementPolicy policy = parse(sb.toString());
+ assertEquals("root.parentq.user1",
+ policy.assignAppToQueue("root.default", "user1"));
+ }
+
private QueuePlacementPolicy parse(String str) throws Exception {
// Read and parse the allocations file.
DocumentBuilderFactory docBuilderFactory = DocumentBuilderFactory
Modified:
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm?rev=1597204&r1=1597203&r2=1597204&view=diff
==============================================================================
---
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm
(original)
+++
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm
Fri May 23 22:38:52 2014
@@ -310,7 +310,8 @@ Allocation file format
to âparentâ or by configuring at least one leaf under that queue
which makes it a parent.
See example allocation for a sample use case.
- * default: the app is placed into the queue named "default".
+ * default: the app is placed into the queue specified in the âqueueâ
attribute of the
+ default rule. If âqueueâ attribute is not specified, the app is
placed into âroot.defaultâ queue.
* reject: the app is rejected.
@@ -348,7 +349,7 @@ Allocation file format
<rule name=ânestedUserQueueâ>
<rule name=âsecondaryGroupExistingQueueâ create=âfalseâ />
</rule>
- <rule name="default" />
+ <rule name="default" queue=âsample_queueâ />
</queuePlacementPolicy>
</allocations>
---
