Author: sandy
Date: Thu Oct 3 01:21:47 2013
New Revision: 1528697
URL: http://svn.apache.org/r1528697
Log:
YARN-1213. Restore config to ban submitting to undeclared pools in the Fair
Scheduler. (Sandy Ryza)
Modified:
hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt?rev=1528697&r1=1528696&r2=1528697&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt Thu Oct 3
01:21:47 2013
@@ -77,6 +77,9 @@ Release 2.1.2 - UNRELEASED
YARN-1228. Clean up Fair Scheduler configuration loading. (Sandy Ryza)
+ YARN-1213. Restore config to ban submitting to undeclared pools in the
+ Fair Scheduler. (Sandy Ryza)
+
OPTIMIZATIONS
BUG FIXES
Modified:
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java?rev=1528697&r1=1528696&r2=1528697&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
Thu Oct 3 01:21:47 2013
@@ -677,10 +677,11 @@ public class FairScheduler implements Re
queueName = user;
}
- FSLeafQueue queue = queueMgr.getLeafQueue(queueName);
+ FSLeafQueue queue = queueMgr.getLeafQueue(queueName,
+ conf.getAllowUndeclaredPools());
if (queue == null) {
// queue is not an existing or createable leaf queue
- queue = queueMgr.getLeafQueue(YarnConfiguration.DEFAULT_QUEUE_NAME);
+ queue = queueMgr.getLeafQueue(YarnConfiguration.DEFAULT_QUEUE_NAME,
false);
}
if (rmApp != null) {
@@ -726,7 +727,7 @@ public class FairScheduler implements Re
// Inform the queue
FSLeafQueue queue = queueMgr.getLeafQueue(application.getQueue()
- .getQueueName());
+ .getQueueName(), false);
queue.removeApp(application);
// Remove from our data-structure
Modified:
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java?rev=1528697&r1=1528696&r2=1528697&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java
Thu Oct 3 01:21:47 2013
@@ -54,6 +54,11 @@ public class FairSchedulerConfiguration
protected static final String DEFAULT_ALLOCATION_FILE = "fair-scheduler.xml";
protected static final String EVENT_LOG_DIR = "eventlog.dir";
+ /** Whether pools can be created that were not specified in the FS
configuration file
+ */
+ protected static final String ALLOW_UNDECLARED_POOLS = CONF_PREFIX +
"allow-undeclared-pools";
+ protected static final boolean DEFAULT_ALLOW_UNDECLARED_POOLS = true;
+
/** Whether to use the user name as the queue name (instead of "default") if
* the request does not specify a queue. */
protected static final String USER_AS_DEFAULT_QUEUE = CONF_PREFIX +
"user-as-default-queue";
@@ -141,6 +146,10 @@ public class FairSchedulerConfiguration
DEFAULT_RM_SCHEDULER_INCREMENT_ALLOCATION_VCORES);
return Resources.createResource(incrementMemory, incrementCores);
}
+
+ public boolean getAllowUndeclaredPools() {
+ return getBoolean(ALLOW_UNDECLARED_POOLS, DEFAULT_ALLOW_UNDECLARED_POOLS);
+ }
public boolean getUserAsDefaultQueue() {
return getBoolean(USER_AS_DEFAULT_QUEUE, DEFAULT_USER_AS_DEFAULT_QUEUE);
Modified:
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java?rev=1528697&r1=1528696&r2=1528697&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java
Thu Oct 3 01:21:47 2013
@@ -109,26 +109,27 @@ public class QueueManager {
lastSuccessfulReload = scheduler.getClock().getTime();
lastReloadAttempt = scheduler.getClock().getTime();
// Create the default queue
- getLeafQueue(YarnConfiguration.DEFAULT_QUEUE_NAME);
+ getLeafQueue(YarnConfiguration.DEFAULT_QUEUE_NAME, true);
}
/**
- * Get a queue by name, creating it if necessary. If the queue
- * is not or can not be a leaf queue, i.e. it already exists as a parent
queue,
- * or one of the parents in its name is already a leaf queue, null is
returned.
+ * Get a queue by name, creating it if the create param is true and is
necessary.
+ * If the queue is not or can not be a leaf queue, i.e. it already exists as
a
+ * parent queue, or one of the parents in its name is already a leaf queue,
+ * null is returned.
*
* The root part of the name is optional, so a queue underneath the root
* named "queue1" could be referred to as just "queue1", and a queue named
* "queue2" underneath a parent named "parent1" that is underneath the root
* could be referred to as just "parent1.queue2".
*/
- public FSLeafQueue getLeafQueue(String name) {
+ public FSLeafQueue getLeafQueue(String name, boolean create) {
if (!name.startsWith(ROOT_QUEUE + ".")) {
name = ROOT_QUEUE + "." + name;
}
synchronized (queues) {
FSQueue queue = queues.get(name);
- if (queue == null) {
+ if (queue == null && create) {
FSLeafQueue leafQueue = createLeafQueue(name);
if (leafQueue == null) {
return null;
@@ -224,13 +225,6 @@ public class QueueManager {
}
/**
- * Get the queue for a given AppSchedulable.
- */
- public FSLeafQueue getQueueForApp(AppSchedulable app) {
- return getLeafQueue(app.getApp().getQueueName());
- }
-
- /**
* Reload allocations file if it hasn't been loaded in a while
*/
public void reloadAllocsIfNecessary() {
@@ -384,7 +378,7 @@ public class QueueManager {
// Create all queus
for (String name: queueNamesInAllocFile) {
- getLeafQueue(name);
+ getLeafQueue(name, true);
}
// Set custom policies as specified
Modified:
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java?rev=1528697&r1=1528696&r2=1528697&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
Thu Oct 3 01:21:47 2013
@@ -436,9 +436,9 @@ public class TestFairScheduler {
Collection<FSLeafQueue> queues = queueManager.getLeafQueues();
assertEquals(3, queues.size());
- FSLeafQueue queue1 = queueManager.getLeafQueue("default");
- FSLeafQueue queue2 = queueManager.getLeafQueue("parent.queue2");
- FSLeafQueue queue3 = queueManager.getLeafQueue("parent.queue3");
+ FSLeafQueue queue1 = queueManager.getLeafQueue("default", true);
+ FSLeafQueue queue2 = queueManager.getLeafQueue("parent.queue2", true);
+ FSLeafQueue queue3 = queueManager.getLeafQueue("parent.queue3", true);
assertEquals(capacity / 2, queue1.getFairShare().getMemory());
assertEquals(capacity / 2, queue1.getMetrics().getFairShareMB());
assertEquals(capacity / 4, queue2.getFairShare().getMemory());
@@ -450,20 +450,20 @@ public class TestFairScheduler {
@Test
public void testHierarchicalQueuesSimilarParents() {
QueueManager queueManager = scheduler.getQueueManager();
- FSLeafQueue leafQueue = queueManager.getLeafQueue("parent.child");
+ FSLeafQueue leafQueue = queueManager.getLeafQueue("parent.child", true);
Assert.assertEquals(2, queueManager.getLeafQueues().size());
Assert.assertNotNull(leafQueue);
Assert.assertEquals("root.parent.child", leafQueue.getName());
- FSLeafQueue leafQueue2 = queueManager.getLeafQueue("parent");
+ FSLeafQueue leafQueue2 = queueManager.getLeafQueue("parent", true);
Assert.assertNull(leafQueue2);
Assert.assertEquals(2, queueManager.getLeafQueues().size());
- FSLeafQueue leafQueue3 =
queueManager.getLeafQueue("parent.child.grandchild");
+ FSLeafQueue leafQueue3 =
queueManager.getLeafQueue("parent.child.grandchild", true);
Assert.assertNull(leafQueue3);
Assert.assertEquals(2, queueManager.getLeafQueues().size());
- FSLeafQueue leafQueue4 = queueManager.getLeafQueue("parent.sister");
+ FSLeafQueue leafQueue4 = queueManager.getLeafQueue("parent.sister", true);
Assert.assertNotNull(leafQueue4);
Assert.assertEquals("root.parent.sister", leafQueue4.getName());
Assert.assertEquals(3, queueManager.getLeafQueues().size());
@@ -612,9 +612,9 @@ public class TestFairScheduler {
AppAddedSchedulerEvent appAddedEvent = new AppAddedSchedulerEvent(
createAppAttemptId(1, 1), "default", "user1");
scheduler.handle(appAddedEvent);
- assertEquals(1, scheduler.getQueueManager().getLeafQueue("user1")
+ assertEquals(1, scheduler.getQueueManager().getLeafQueue("user1", true)
.getAppSchedulables().size());
- assertEquals(0, scheduler.getQueueManager().getLeafQueue("default")
+ assertEquals(0, scheduler.getQueueManager().getLeafQueue("default", true)
.getAppSchedulables().size());
conf.set(FairSchedulerConfiguration.USER_AS_DEFAULT_QUEUE, "false");
@@ -622,11 +622,11 @@ public class TestFairScheduler {
AppAddedSchedulerEvent appAddedEvent2 = new AppAddedSchedulerEvent(
createAppAttemptId(2, 1), "default", "user2");
scheduler.handle(appAddedEvent2);
- assertEquals(1, scheduler.getQueueManager().getLeafQueue("user1")
+ assertEquals(1, scheduler.getQueueManager().getLeafQueue("user1", true)
.getAppSchedulables().size());
- assertEquals(1, scheduler.getQueueManager().getLeafQueue("default")
+ assertEquals(1, scheduler.getQueueManager().getLeafQueue("default", true)
.getAppSchedulables().size());
- assertEquals(0, scheduler.getQueueManager().getLeafQueue("user2")
+ assertEquals(0, scheduler.getQueueManager().getLeafQueue("user2", true)
.getAppSchedulables().size());
}
@@ -772,7 +772,7 @@ public class TestFairScheduler {
assertEquals(2, scheduler.getQueueManager().getLeafQueues().size());
// That queue should have one app
- assertEquals(1, scheduler.getQueueManager().getLeafQueue("user1")
+ assertEquals(1, scheduler.getQueueManager().getLeafQueue("user1", true)
.getAppSchedulables().size());
AppRemovedSchedulerEvent appRemovedEvent1 = new AppRemovedSchedulerEvent(
@@ -782,7 +782,7 @@ public class TestFairScheduler {
scheduler.handle(appRemovedEvent1);
// Queue should have no apps
- assertEquals(0, scheduler.getQueueManager().getLeafQueue("user1")
+ assertEquals(0, scheduler.getQueueManager().getLeafQueue("user1", true)
.getAppSchedulables().size());
}
@@ -919,10 +919,10 @@ public class TestFairScheduler {
Collection<FSLeafQueue> leafQueues = queueManager.getLeafQueues();
Assert.assertEquals(4, leafQueues.size());
- Assert.assertNotNull(queueManager.getLeafQueue("queueA"));
- Assert.assertNotNull(queueManager.getLeafQueue("queueB.queueC"));
- Assert.assertNotNull(queueManager.getLeafQueue("queueB.queueD"));
- Assert.assertNotNull(queueManager.getLeafQueue("default"));
+ Assert.assertNotNull(queueManager.getLeafQueue("queueA", true));
+ Assert.assertNotNull(queueManager.getLeafQueue("queueB.queueC", true));
+ Assert.assertNotNull(queueManager.getLeafQueue("queueB.queueD", true));
+ Assert.assertNotNull(queueManager.getLeafQueue("default", true));
// Make sure querying for queues didn't create any new ones:
Assert.assertEquals(4, leafQueues.size());
}
@@ -1423,9 +1423,9 @@ public class TestFairScheduler {
scheduler.update();
FSLeafQueue schedC =
- scheduler.getQueueManager().getLeafQueue("queueC");
+ scheduler.getQueueManager().getLeafQueue("queueC", true);
FSLeafQueue schedD =
- scheduler.getQueueManager().getLeafQueue("queueD");
+ scheduler.getQueueManager().getLeafQueue("queueD", true);
assertTrue(Resources.equals(
Resources.none(), scheduler.resToPreempt(schedC, clock.getTime())));
@@ -1688,7 +1688,7 @@ public class TestFairScheduler {
FSSchedulerApp app1 = scheduler.applications.get(attId1);
FSSchedulerApp app2 = scheduler.applications.get(attId2);
- FSLeafQueue queue1 = scheduler.getQueueManager().getLeafQueue("queue1");
+ FSLeafQueue queue1 = scheduler.getQueueManager().getLeafQueue("queue1",
true);
queue1.setPolicy(new FifoPolicy());
scheduler.update();
@@ -1716,7 +1716,7 @@ public class TestFairScheduler {
public void testMaxAssign() throws AllocationConfigurationException {
// set required scheduler configs
scheduler.assignMultiple = true;
- scheduler.getQueueManager().getLeafQueue("root.default")
+ scheduler.getQueueManager().getLeafQueue("root.default", true)
.setPolicy(SchedulingPolicy.getDefault());
RMNode node =
@@ -1793,7 +1793,7 @@ public class TestFairScheduler {
FSSchedulerApp app3 = scheduler.applications.get(attId3);
FSSchedulerApp app4 = scheduler.applications.get(attId4);
- scheduler.getQueueManager().getLeafQueue(fifoQueue)
+ scheduler.getQueueManager().getLeafQueue(fifoQueue, true)
.setPolicy(SchedulingPolicy.parse("fifo"));
scheduler.update();
@@ -2405,4 +2405,46 @@ public class TestFairScheduler {
//expected
}
}
+
+ @Test
+ public void testDontAllowUndeclaredPools() throws Exception{
+ Configuration conf = createConfiguration();
+ conf.setBoolean(FairSchedulerConfiguration.ALLOW_UNDECLARED_POOLS, false);
+ conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
+ scheduler.reinitialize(conf, resourceManager.getRMContext());
+
+ PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
+ out.println("<?xml version=\"1.0\"?>");
+ out.println("<allocations>");
+ out.println("<queue name=\"jerry\">");
+ out.println("</queue>");
+ out.println("</allocations>");
+ out.close();
+
+ QueueManager queueManager = scheduler.getQueueManager();
+ queueManager.initialize();
+
+ FSLeafQueue jerryQueue = queueManager.getLeafQueue("jerry", false);
+ FSLeafQueue defaultQueue = queueManager.getLeafQueue("default", false);
+
+ // Should get put into jerry
+ createSchedulingRequest(1024, "jerry", "someuser");
+ assertEquals(1, jerryQueue.getAppSchedulables().size());
+
+ // Should get forced into default
+ createSchedulingRequest(1024, "newqueue", "someuser");
+ assertEquals(1, jerryQueue.getAppSchedulables().size());
+ assertEquals(1, defaultQueue.getAppSchedulables().size());
+
+ // Would get put into someuser because of user-as-default-queue, but should
+ // be forced into default
+ createSchedulingRequest(1024, "default", "someuser");
+ assertEquals(1, jerryQueue.getAppSchedulables().size());
+ assertEquals(2, defaultQueue.getAppSchedulables().size());
+
+ // Should get put into jerry because of user-as-default-queue
+ createSchedulingRequest(1024, "default", "jerry");
+ assertEquals(2, jerryQueue.getAppSchedulables().size());
+ assertEquals(2, defaultQueue.getAppSchedulables().size());
+ }
}
