Author: vinodkv
Date: Mon Mar 10 20:37:09 2014
New Revision: 1576073
URL: http://svn.apache.org/r1576073
Log:
YARN-1788. Fixed a bug in ResourceManager to set the apps-completed and
apps-killed metrics correctly for killed applications. Contributed by Varun
Vasudev.
svn merge --ignore-ancestry -c 1576072 ../../trunk/
Modified:
hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRM.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt?rev=1576073&r1=1576072&r2=1576073&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt Mon Mar 10
20:37:09 2014
@@ -421,6 +421,10 @@ Release 2.4.0 - UNRELEASED
YARN-1793. Fixed ClientRMService#forceKillApplication not killing unmanaged
application. (Karthik Kambatla via jianhe)
+ YARN-1788. Fixed a bug in ResourceManager to set the apps-completed and
+ apps-killed metrics correctly for killed applications. (Varun Vasudev via
+ vinodkv)
+
Release 2.3.1 - UNRELEASED
INCOMPATIBLE CHANGES
Modified:
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java?rev=1576073&r1=1576072&r2=1576073&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
Mon Mar 10 20:37:09 2014
@@ -1043,8 +1043,8 @@ public class RMAppImpl implements RMApp,
if (app.finishTime == 0 ) {
app.finishTime = System.currentTimeMillis();
}
- app.handler.handle(new AppRemovedSchedulerEvent(app.applicationId, app
- .getState()));
+ app.handler.handle(new AppRemovedSchedulerEvent(app.applicationId,
+ finalState));
app.handler.handle(
new RMAppManagerEvent(app.applicationId,
RMAppManagerEventType.APP_COMPLETED));
Modified:
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRM.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRM.java?rev=1576073&r1=1576072&r2=1576073&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRM.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRM.java
Mon Mar 10 20:37:09 2014
@@ -62,6 +62,7 @@ import org.apache.hadoop.yarn.server.res
import
org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptEventType;
import
org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
import
org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM;
import org.apache.log4j.Level;
import org.apache.log4j.LogManager;
@@ -519,7 +520,13 @@ public class TestRM {
}
};
+ // test metrics
+ QueueMetrics metrics = rm.getResourceScheduler().getRootQueueMetrics();
+ int appsKilled = metrics.getAppsKilled();
+ int appsSubmitted = metrics.getAppsSubmitted();
+
rm.start();
+
MockNM nm1 =
new MockNM("127.0.0.1:1234", 15120, rm.getResourceTrackerService());
nm1.registerNode();
@@ -552,6 +559,11 @@ public class TestRM {
new RMAppEvent(application.getApplicationId(),
RMAppEventType.ATTEMPT_KILLED));
rm.waitForState(application.getApplicationId(), RMAppState.KILLED);
+
+ // test metrics
+ metrics = rm.getResourceScheduler().getRootQueueMetrics();
+ Assert.assertEquals(appsKilled + 1, metrics.getAppsKilled());
+ Assert.assertEquals(appsSubmitted + 1, metrics.getAppsSubmitted());
}
public static void main(String[] args) throws Exception {
Modified:
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java?rev=1576073&r1=1576072&r2=1576073&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java
Mon Mar 10 20:37:09 2014
@@ -60,6 +60,7 @@ import org.apache.hadoop.yarn.server.res
import
org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptUpdateSavedEvent;
import
org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.ContainerAllocationExpirer;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler;
+import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppRemovedSchedulerEvent;
import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent;
import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEventType;
import
org.apache.hadoop.yarn.server.resourcemanager.security.AMRMTokenSecretManager;
@@ -87,6 +88,7 @@ public class TestRMAppTransitions {
private RMStateStore store;
private RMApplicationHistoryWriter writer;
private YarnScheduler scheduler;
+ private TestSchedulerEventDispatcher schedulerDispatcher;
// ignore all the RM application attempt events
private static final class TestApplicationAttemptEventDispatcher implements
@@ -148,8 +150,11 @@ public class TestRMAppTransitions {
// handle all the scheduler events - same as in ResourceManager.java
private static final class TestSchedulerEventDispatcher implements
EventHandler<SchedulerEvent> {
+ public SchedulerEvent lastSchedulerEvent;
+
@Override
public void handle(SchedulerEvent event) {
+ lastSchedulerEvent = event;
}
}
@@ -201,8 +206,9 @@ public class TestRMAppTransitions {
rmDispatcher.register(RMAppManagerEventType.class,
new TestApplicationManagerEventDispatcher());
+ schedulerDispatcher = new TestSchedulerEventDispatcher();
rmDispatcher.register(SchedulerEventType.class,
- new TestSchedulerEventDispatcher());
+ schedulerDispatcher);
rmDispatcher.init(conf);
rmDispatcher.start();
@@ -502,6 +508,7 @@ public class TestRMAppTransitions {
assertKilled(application);
assertAppFinalStateNotSaved(application);
verifyApplicationFinished(RMAppState.KILLED);
+ verifyAppRemovedSchedulerEvent(RMAppState.KILLED);
}
@Test
@@ -534,6 +541,7 @@ public class TestRMAppTransitions {
sendAppUpdateSavedEvent(application);
assertKilled(application);
verifyApplicationFinished(RMAppState.KILLED);
+ verifyAppRemovedSchedulerEvent(RMAppState.KILLED);
}
@Test (timeout = 30000)
@@ -583,6 +591,7 @@ public class TestRMAppTransitions {
assertKilled(application);
assertAppFinalStateSaved(application);
verifyApplicationFinished(RMAppState.KILLED);
+ verifyAppRemovedSchedulerEvent(RMAppState.KILLED);
}
@Test
@@ -640,6 +649,7 @@ public class TestRMAppTransitions {
assertKilled(application);
assertAppFinalStateSaved(application);
verifyApplicationFinished(RMAppState.KILLED);
+ verifyAppRemovedSchedulerEvent(RMAppState.KILLED);
}
@Test
@@ -663,6 +673,7 @@ public class TestRMAppTransitions {
sendAppUpdateSavedEvent(application);
assertKilled(application);
verifyApplicationFinished(RMAppState.KILLED);
+ verifyAppRemovedSchedulerEvent(RMAppState.KILLED);
}
@Test
@@ -868,4 +879,15 @@ public class TestRMAppTransitions {
verify(writer).applicationFinished(any(RMApp.class), finalState.capture());
Assert.assertEquals(state, finalState.getValue());
}
+
+ private void verifyAppRemovedSchedulerEvent(RMAppState finalState) {
+ Assert.assertEquals(SchedulerEventType.APP_REMOVED,
+ schedulerDispatcher.lastSchedulerEvent.getType());
+ if(schedulerDispatcher.lastSchedulerEvent instanceof
+ AppRemovedSchedulerEvent) {
+ AppRemovedSchedulerEvent appRemovedEvent =
+ (AppRemovedSchedulerEvent) schedulerDispatcher.lastSchedulerEvent;
+ Assert.assertEquals(finalState, appRemovedEvent.getFinalState());
+ }
+ }
}
