Author: kasha
Date: Fri Mar 7 21:36:43 2014
New Revision: 1575419
URL: http://svn.apache.org/r1575419
Log:
YARN-1774. FS: Submitting to non-leaf queue throws NPE. (Anubhav Dhoot and
Karthik Kambatla via kasha)
Modified:
hadoop/common/branches/branch-2.4/hadoop-yarn-project/CHANGES.txt
hadoop/common/branches/branch-2.4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
hadoop/common/branches/branch-2.4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
Modified: hadoop/common/branches/branch-2.4/hadoop-yarn-project/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.4/hadoop-yarn-project/CHANGES.txt?rev=1575419&r1=1575418&r2=1575419&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.4/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/branches/branch-2.4/hadoop-yarn-project/CHANGES.txt Fri Mar
7 21:36:43 2014
@@ -371,6 +371,8 @@ Release 2.4.0 - UNRELEASED
YARN-1752. Fixed ApplicationMasterService to reject unregister request
if AM did not register before. (Rohith Sharma via jianhe)
+ YARN-1774. FS: Submitting to non-leaf queue throws NPE. (Anubhav Dhoot and
+ Karthik Kambatla via kasha)
Release 2.3.1 - UNRELEASED
Modified:
hadoop/common/branches/branch-2.4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java?rev=1575419&r1=1575418&r2=1575419&view=diff
==============================================================================
---
hadoop/common/branches/branch-2.4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
(original)
+++
hadoop/common/branches/branch-2.4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
Fri Mar 7 21:36:43 2014
@@ -611,9 +611,6 @@ public class FairScheduler extends Abstr
RMApp rmApp = rmContext.getRMApps().get(applicationId);
FSLeafQueue queue = assignToQueue(rmApp, queueName, user);
if (queue == null) {
- rmContext.getDispatcher().getEventHandler().handle(
- new RMAppRejectedEvent(applicationId,
- "Application rejected by queue placement policy"));
return;
}
@@ -679,27 +676,43 @@ public class FairScheduler extends Abstr
new RMAppAttemptEvent(applicationAttemptId,
RMAppAttemptEventType.ATTEMPT_ADDED));
}
-
+
+ /**
+ * Helper method that attempts to assign the app to a queue. The method is
+ * responsible to call the appropriate event-handler if the app is rejected.
+ */
@VisibleForTesting
FSLeafQueue assignToQueue(RMApp rmApp, String queueName, String user) {
FSLeafQueue queue = null;
+ String appRejectMsg = null;
+
try {
QueuePlacementPolicy placementPolicy = allocConf.getPlacementPolicy();
queueName = placementPolicy.assignAppToQueue(queueName, user);
if (queueName == null) {
- return null;
+ appRejectMsg = "Application rejected by queue placement policy";
+ } else {
+ queue = queueMgr.getLeafQueue(queueName, true);
+ if (queue == null) {
+ appRejectMsg = queueName + " is not a leaf queue";
+ }
}
- queue = queueMgr.getLeafQueue(queueName, true);
- } catch (IOException ex) {
- LOG.error("Error assigning app to queue, rejecting", ex);
+ } catch (IOException ioe) {
+ appRejectMsg = "Error assigning app to queue " + queueName;
}
-
+
+ if (appRejectMsg != null && rmApp != null) {
+ LOG.error(appRejectMsg);
+ rmContext.getDispatcher().getEventHandler().handle(
+ new RMAppRejectedEvent(rmApp.getApplicationId(), appRejectMsg));
+ return null;
+ }
+
if (rmApp != null) {
rmApp.setQueue(queue.getName());
} else {
- LOG.warn("Couldn't find RM app to set queue name on");
+ LOG.error("Couldn't find RM app to set queue name on");
}
-
return queue;
}
Modified:
hadoop/common/branches/branch-2.4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java?rev=1575419&r1=1575418&r2=1575419&view=diff
==============================================================================
---
hadoop/common/branches/branch-2.4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
(original)
+++
hadoop/common/branches/branch-2.4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
Fri Mar 7 21:36:43 2014
@@ -704,6 +704,22 @@ public class TestFairScheduler {
assertEquals(rmApp2.getQueue(), queue2.getName());
assertEquals("root.notdefault", rmApp2.getQueue());
}
+
+ @Test
+ public void testAssignToNonLeafQueueReturnsNull() throws Exception {
+ conf.set(FairSchedulerConfiguration.USER_AS_DEFAULT_QUEUE, "true");
+ scheduler.reinitialize(conf, resourceManager.getRMContext());
+
+ scheduler.getQueueManager().getLeafQueue("root.child1.granchild", true);
+ scheduler.getQueueManager().getLeafQueue("root.child2", true);
+
+ RMApp rmApp1 = new MockRMApp(0, 0, RMAppState.NEW);
+ RMApp rmApp2 = new MockRMApp(1, 1, RMAppState.NEW);
+
+ // Trying to assign to non leaf queue would return null
+ assertNull(scheduler.assignToQueue(rmApp1, "root.child1", "tintin"));
+ assertNotNull(scheduler.assignToQueue(rmApp2, "root.child2", "snowy"));
+ }
@Test
public void testQueuePlacementWithPolicy() throws Exception {
