Author: tucu
Date: Thu Jul 11 21:39:28 2013
New Revision: 1502374
URL: http://svn.apache.org/r1502374
Log:
YARN-333. Schedulers cannot control the queue-name of an application. (sandyr
via tucu)
Modified:
hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
Modified: hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt?rev=1502374&r1=1502373&r2=1502374&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt Thu Jul 11 21:39:28 2013
@@ -751,6 +751,9 @@ Release 2.1.0-beta - 2013-07-02
YARN-799. Fix CgroupsLCEResourcesHandler to use /tasks instead of
/cgroup.procs. (Chris Riccomini via acmurthy)
+ YARN-333. Schedulers cannot control the queue-name of an
+ application. (sandyr via tucu)
+
Release 2.0.5-alpha - 06/06/2013
INCOMPATIBLE CHANGES
Modified:
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java?rev=1502374&r1=1502373&r2=1502374&view=diff
==============================================================================
---
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java
(original)
+++
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java
Thu Jul 11 21:39:28 2013
@@ -33,7 +33,7 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
/**
- * The read interface to an Application in the ResourceManager. Take a
+ * The interface to an Application in the ResourceManager. Take a
* look at {@link RMAppImpl} for its implementation. This interface
* exposes methods to access various updates in application status/report.
*/
@@ -85,6 +85,13 @@ public interface RMApp extends EventHand
* @return the queue to which the application was submitted to.
*/
String getQueue();
+
+ /**
+ * Reflects a change in the application's queue from the one specified in the
+ * {@link ApplicationSubmissionContext}.
+ * @param name the new queue name
+ */
+ void setQueue(String name);
/**
* The name of the application as set in {@link
Modified:
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java?rev=1502374&r1=1502373&r2=1502374&view=diff
==============================================================================
---
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
(original)
+++
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
Thu Jul 11 21:39:28 2013
@@ -81,7 +81,6 @@ public class RMAppImpl implements RMApp,
private final RMContext rmContext;
private final Configuration conf;
private final String user;
- private final String queue;
private final String name;
private final ApplicationSubmissionContext submissionContext;
private final Dispatcher dispatcher;
@@ -101,6 +100,7 @@ public class RMAppImpl implements RMApp,
private long startTime;
private long finishTime;
private RMAppAttempt currentAttempt;
+ private String queue;
@SuppressWarnings("rawtypes")
private EventHandler handler;
private static final FinalTransition FINAL_TRANSITION = new
FinalTransition();
@@ -341,6 +341,11 @@ public class RMAppImpl implements RMApp,
public String getQueue() {
return this.queue;
}
+
+ @Override
+ public void setQueue(String queue) {
+ this.queue = queue;
+ }
@Override
public String getName() {
Modified:
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java?rev=1502374&r1=1502373&r2=1502374&view=diff
==============================================================================
---
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
(original)
+++
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
Thu Jul 11 21:39:28 2013
@@ -57,6 +57,7 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import
org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.RMState;
import org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceWeights;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import
org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptEvent;
import
org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptEventType;
import
org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
@@ -88,6 +89,8 @@ import org.apache.hadoop.yarn.util.resou
import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
import org.apache.hadoop.yarn.util.resource.Resources;
+import com.google.common.annotations.VisibleForTesting;
+
/**
* A scheduler that schedules resources between a set of queues. The scheduler
* keeps track of the resources used by each queue, and attempts to maintain
@@ -601,12 +604,8 @@ public class FairScheduler implements Re
*/
protected synchronized void addApplication(
ApplicationAttemptId applicationAttemptId, String queueName, String
user) {
-
- FSLeafQueue queue = queueMgr.getLeafQueue(queueName);
- if (queue == null) {
- // queue is not an existing or createable leaf queue
- queue = queueMgr.getLeafQueue(YarnConfiguration.DEFAULT_QUEUE_NAME);
- }
+ RMApp rmApp = rmContext.getRMApps().get(applicationAttemptId);
+ FSLeafQueue queue = assignToQueue(rmApp, queueName, user);
FSSchedulerApp schedulerApp =
new FSSchedulerApp(applicationAttemptId, user,
@@ -637,6 +636,27 @@ public class FairScheduler implements Re
new RMAppAttemptEvent(applicationAttemptId,
RMAppAttemptEventType.APP_ACCEPTED));
}
+
+ @VisibleForTesting
+ FSLeafQueue assignToQueue(RMApp rmApp, String queueName, String user) {
+ // Potentially set queue to username if configured to do so
+ if (queueName.equals(YarnConfiguration.DEFAULT_QUEUE_NAME) &&
+ userAsDefaultQueue) {
+ queueName = user;
+ }
+
+ FSLeafQueue queue = queueMgr.getLeafQueue(queueName);
+ if (queue == null) {
+ // queue is not an existing or createable leaf queue
+ queue = queueMgr.getLeafQueue(YarnConfiguration.DEFAULT_QUEUE_NAME);
+ }
+
+ if (rmApp != null) {
+ rmApp.setQueue(queue.getName());
+ }
+
+ return queue;
+ }
private synchronized void removeApplication(
ApplicationAttemptId applicationAttemptId,
@@ -985,13 +1005,6 @@ public class FairScheduler implements Re
}
AppAddedSchedulerEvent appAddedEvent = (AppAddedSchedulerEvent)event;
String queue = appAddedEvent.getQueue();
-
- // Potentially set queue to username if configured to do so
- String def = YarnConfiguration.DEFAULT_QUEUE_NAME;
- if (queue.equals(def) && userAsDefaultQueue) {
- queue = appAddedEvent.getUser();
- }
-
addApplication(appAddedEvent.getApplicationAttemptId(), queue,
appAddedEvent.getUser());
break;
Modified:
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java?rev=1502374&r1=1502373&r2=1502374&view=diff
==============================================================================
---
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java
(original)
+++
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java
Thu Jul 11 21:39:28 2013
@@ -136,6 +136,11 @@ public abstract class MockAsm extends Mo
public String getApplicationType() {
throw new UnsupportedOperationException("Not supported yet.");
}
+
+ @Override
+ public void setQueue(String name) {
+ throw new UnsupportedOperationException("Not supported yet.");
+ }
}
public static RMApp newApplication(int i) {
Modified:
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java?rev=1502374&r1=1502373&r2=1502374&view=diff
==============================================================================
---
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
(original)
+++
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
Thu Jul 11 21:39:28 2013
@@ -63,6 +63,7 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.MockNodes;
import org.apache.hadoop.yarn.server.resourcemanager.RMContextImpl;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.MockRMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEventType;
@@ -393,7 +394,6 @@ public class TestFairScheduler {
scheduler.handle(nodeEvent1);
// Have two queues which want entire cluster capacity
- createSchedulingRequest(10 * 1024, "default", "user1");
createSchedulingRequest(10 * 1024, "parent.queue2", "user1");
createSchedulingRequest(10 * 1024, "parent.queue3", "user1");
@@ -558,6 +558,25 @@ public class TestFairScheduler {
assertEquals(0, scheduler.getQueueManager().getLeafQueue("user2")
.getAppSchedulables().size());
}
+
+ @Test
+ public void testAssignToQueue() throws Exception {
+ Configuration conf = createConfiguration();
+ conf.set(FairSchedulerConfiguration.USER_AS_DEFAULT_QUEUE, "true");
+ scheduler.reinitialize(conf, resourceManager.getRMContext());
+
+ RMApp rmApp1 = new MockRMApp(0, 0, RMAppState.NEW);
+ RMApp rmApp2 = new MockRMApp(1, 1, RMAppState.NEW);
+
+ FSLeafQueue queue1 = scheduler.assignToQueue(rmApp1, "default", "asterix");
+ FSLeafQueue queue2 = scheduler.assignToQueue(rmApp2, "notdefault",
"obelix");
+
+ // assert FSLeafQueue's name is the correct name is the one set in the
RMApp
+ assertEquals(rmApp1.getQueue(), queue1.getName());
+ assertEquals("root.asterix", rmApp1.getQueue());
+ assertEquals(rmApp2.getQueue(), queue2.getName());
+ assertEquals("root.notdefault", rmApp2.getQueue());
+ }
@Test
public void testFairShareWithMinAlloc() throws Exception {
