Author: mahadev
Date: Tue May 10 21:25:15 2011
New Revision: 1101651
URL: http://svn.apache.org/viewvc?rev=1101651&view=rev
Log:
Propagate error back to client in case of a job submission failure (mahadev)
Modified:
hadoop/mapreduce/branches/MR-279/CHANGES.txt
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/AMTracker.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/ApplicationMasterInfo.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/ApplicationsManagerImpl.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestApplicationMasterExpiry.java
Modified: hadoop/mapreduce/branches/MR-279/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/CHANGES.txt?rev=1101651&r1=1101650&r2=1101651&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/CHANGES.txt (original)
+++ hadoop/mapreduce/branches/MR-279/CHANGES.txt Tue May 10 21:25:15 2011
@@ -4,6 +4,8 @@ Trunk (unreleased changes)
MAPREDUCE-279
+ Propagate error back to client in case of a job submission failure
(mahadev)
+
MAPREDUCE-2480: MR App should not depend on hard coded version of shuffle
(luke lu via mahadev)
Fix NM to use multiple disks for local files and the userlogs. (vinodkv)
Modified:
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/AMTracker.java
URL:
http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/AMTracker.java?rev=1101651&r1=1101650&r2=1101651&view=diff
==============================================================================
---
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/AMTracker.java
(original)
+++
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/AMTracker.java
Tue May 10 21:25:15 2011
@@ -223,10 +223,27 @@ public class AMTracker extends AbstractS
synchronized(applications) {
applications.put(applicationMaster.getApplicationID(),
applicationMaster);
}
+
+ }
+
+ public void runApplication(ApplicationId applicationId) {
+ ApplicationMasterInfo masterInfo = null;
+ synchronized (applications) {
+ masterInfo = applications.get(applicationId);
+ }
rmContext.getDispatcher().getSyncHandler().handle(new
ASMEvent<ApplicationEventType>(
- ApplicationEventType.ALLOCATE, applicationMaster));
-
+ ApplicationEventType.ALLOCATE, masterInfo));
+
}
+
+ public void finishNonRunnableApplication(ApplicationId applicationId) {
+ ApplicationMasterInfo masterInfo = null;
+ synchronized (applications) {
+ masterInfo = applications.get(applicationId);
+ }
+ rmContext.getDispatcher().getSyncHandler().handle(new
ASMEvent<ApplicationEventType>(
+ ApplicationEventType.FAILED, masterInfo));
+ }
public void finish(ApplicationId application) {
ApplicationMasterInfo masterInfo = null;
Modified:
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/ApplicationMasterInfo.java
URL:
http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/ApplicationMasterInfo.java?rev=1101651&r1=1101650&r2=1101651&view=diff
==============================================================================
---
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/ApplicationMasterInfo.java
(original)
+++
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/ApplicationMasterInfo.java
Tue May 10 21:25:15 2011
@@ -94,6 +94,9 @@ public class ApplicationMasterInfo imple
.addTransition(ApplicationState.PENDING, ApplicationState.ALLOCATING,
ApplicationEventType.ALLOCATE, allocateTransition)
+ .addTransition(ApplicationState.PENDING, ApplicationState.FAILED,
+ ApplicationEventType.FAILED)
+
.addTransition(ApplicationState.PENDING, ApplicationState.ALLOCATING,
ApplicationEventType.RECOVER, allocateTransition)
Modified:
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/ApplicationsManagerImpl.java
URL:
http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/ApplicationsManagerImpl.java?rev=1101651&r1=1101650&r2=1101651&view=diff
==============================================================================
---
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/ApplicationsManagerImpl.java
(original)
+++
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/ApplicationsManagerImpl.java
Tue May 10 21:25:15 2011
@@ -177,8 +177,21 @@ public class ApplicationsManagerImpl ext
context.setQueue(context.getQueue() == null ? "default" :
context.getQueue());
context.setApplicationName(context.getApplicationName() == null ? "N/A" :
context.getApplicationName());
-
amTracker.addMaster(user, context, clientTokenStr);
+ ApplicationMasterInfo masterInfo = amTracker.get(applicationId);
+ /** this can throw so we need to call it synchronously to let the client
+ * know as soon as it submits. For backwards compatibility we cannot make
+ * it asynchronous
+ */
+ try {
+ scheduler.addApplication(applicationId, masterInfo.getMaster(), user,
masterInfo.getQueue(),
+ context.getPriority(), masterInfo.getStore());
+ } catch(IOException io) {
+ LOG.info("Failed to submit application " + applicationId, io);
+ amTracker.finishNonRunnableApplication(applicationId);
+ throw io;
+ }
+ amTracker.runApplication(applicationId);
// TODO this should happen via dispatcher. should move it out to scheudler
// negotiator.
LOG.info("Application with id " + applicationId.getId() + " submitted by
user " +
Modified:
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
URL:
http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java?rev=1101651&r1=1101650&r2=1101651&view=diff
==============================================================================
---
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
(original)
+++
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
Tue May 10 21:25:15 2011
@@ -283,9 +283,9 @@ implements ResourceScheduler, CapacitySc
// Inform all NodeManagers about completion of application
finishedApplication(applicationId,
application.getAllNodesForApplication());
+ // Remove from our data-structure
+ applications.remove(applicationId);
}
- // Remove from our data-structure
- applications.remove(applicationId);
}
@Override
@@ -466,16 +466,9 @@ implements ResourceScheduler, CapacitySc
public synchronized void handle(ASMEvent<ApplicationTrackerEventType> event)
{
switch(event.getType()) {
case ADD:
- try {
- addApplication(event.getAppContext().getApplicationID(),
event.getAppContext().getMaster(),
- event.getAppContext().getUser(), event.getAppContext().getQueue(),
- event.getAppContext().getSubmissionContext().getPriority(),
- event.getAppContext().getStore());
- } catch(IOException ie) {
- LOG.error("Error in adding an application to the scheduler", ie);
- //TODO do proper error handling to shutdown the Resource Manager is we
- // are not able to handle this.
- }
+ /** ignore add since its called sychronously from the applications
manager
+ *
+ */
break;
case REMOVE:
try {
@@ -488,6 +481,9 @@ implements ResourceScheduler, CapacitySc
break;
case EXPIRE:
try {
+ /** do not remove the application. Just do everything else exception
+ * removing the application
+ */
removeApplication(event.getAppContext().getApplicationID(), false);
} catch(IOException ie) {
LOG.error("Error in removing application", ie);
Modified:
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
URL:
http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java?rev=1101651&r1=1101650&r2=1101651&view=diff
==============================================================================
---
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
(original)
+++
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
Tue May 10 21:25:15 2011
@@ -258,9 +258,9 @@ public class FifoScheduler implements Re
// Let the cluster know that the applications are done
finishedApplication(applicationId,
application.getAllNodesForApplication());
+ // Remove the application
+ applications.remove(applicationId);
}
- // Remove the application
- applications.remove(applicationId);
}
/**
@@ -519,16 +519,9 @@ public class FifoScheduler implements Re
public synchronized void handle(ASMEvent<ApplicationTrackerEventType> event)
{
switch(event.getType()) {
case ADD:
- try {
- addApplication(event.getAppContext().getApplicationID(),
event.getAppContext().getMaster(),
- event.getAppContext().getUser(),
- event.getAppContext().getQueue(),
event.getAppContext().getSubmissionContext().getPriority()
- , event.getAppContext().getStore());
- } catch(IOException ie) {
- LOG.error("Unable to add application " +
event.getAppContext().getApplicationID(), ie);
- /** this is fatal we are not able to add applications for scheduling
**/
- //TODO handle it later.
- }
+ /**
+ * ignore add since its called syncronously from applications manager.
+ */
break;
case REMOVE:
try {
Modified:
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestApplicationMasterExpiry.java
URL:
http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestApplicationMasterExpiry.java?rev=1101651&r1=1101650&r2=1101651&view=diff
==============================================================================
---
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestApplicationMasterExpiry.java
(original)
+++
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestApplicationMasterExpiry.java
Tue May 10 21:25:15 2011
@@ -144,6 +144,7 @@ public class TestApplicationMasterExpiry
"dummy",
context, "dummytoken");
ApplicationMasterInfo masterInfo = tracker.get(context.getApplicationId());
+ tracker.runApplication(context.getApplicationId());
this.context.getDispatcher().getEventHandler().handle(
new ASMEvent<ApplicationEventType>(ApplicationEventType.ALLOCATED,
masterInfo));
waitForState(masterInfo, ApplicationState.LAUNCHING);
