Author: sseth
Date: Fri Feb 24 02:13:25 2012
New Revision: 1293061
URL: http://svn.apache.org/viewvc?rev=1293061&view=rev
Log:
merge MAPREDUCE-3738 from trunk
Modified:
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java
Modified:
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt?rev=1293061&r1=1293060&r2=1293061&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt
(original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt Fri
Feb 24 02:13:25 2012
@@ -45,6 +45,9 @@ Release 0.23.2 - UNRELEASED
MAPREDUCE-3878. Null user on filtered jobhistory job page (Jonathon Eagles
via tgraves)
+
+ MAPREDUCE-3738. MM can hang during shutdown if AppLogAggregatorImpl thread
+ dies unexpectedly (Jason Lowe via sseth)
Release 0.23.1 - 2012-02-17
Modified:
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java?rev=1293061&r1=1293060&r2=1293061&view=diff
==============================================================================
---
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java
(original)
+++
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java
Fri Feb 24 02:13:25 2012
@@ -133,8 +133,16 @@ public class AppLogAggregatorImpl implem
}
@Override
+ public void run() {
+ try {
+ doAppLogAggregation();
+ } finally {
+ this.appAggregationFinished.set(true);
+ }
+ }
+
@SuppressWarnings("unchecked")
- public void run() {
+ private void doAppLogAggregation() {
ContainerId containerId;
while (!this.appFinishing.get()) {
@@ -189,8 +197,6 @@ public class AppLogAggregatorImpl implem
this.dispatcher.getEventHandler().handle(
new ApplicationEvent(this.appId,
ApplicationEventType.APPLICATION_LOG_HANDLING_FINISHED));
-
- this.appAggregationFinished.set(true);
}
private Path getRemoteNodeTmpLogFileForApp() {
Modified:
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java?rev=1293061&r1=1293060&r2=1293061&view=diff
==============================================================================
---
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java
(original)
+++
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java
Fri Feb 24 02:13:25 2012
@@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.server.no
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.when;
import static junit.framework.Assert.assertEquals;
import static junit.framework.Assert.assertTrue;
@@ -69,6 +70,7 @@ import org.apache.hadoop.yarn.logaggrega
import org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat.LogReader;
import org.apache.hadoop.yarn.server.nodemanager.CMgrCompletedAppsEvent;
import org.apache.hadoop.yarn.server.nodemanager.DeletionService;
+import org.apache.hadoop.yarn.server.nodemanager.LocalDirsHandlerService;
import
org.apache.hadoop.yarn.server.nodemanager.containermanager.BaseContainerManagerTest;
import
org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationEvent;
import
org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationEventType;
@@ -536,4 +538,31 @@ public class TestLogAggregationService e
appAcls.put(ApplicationAccessType.VIEW_APP, "*");
return appAcls;
}
+
+ @Test(timeout=20000)
+ @SuppressWarnings("unchecked")
+ public void testStopAfterError() throws Exception {
+ DeletionService delSrvc = mock(DeletionService.class);
+
+ // get the AppLogAggregationImpl thread to crash
+ LocalDirsHandlerService mockedDirSvc = mock(LocalDirsHandlerService.class);
+ when(mockedDirSvc.getLogDirs()).thenThrow(new RuntimeException());
+
+ DrainDispatcher dispatcher = createDispatcher();
+ EventHandler<ApplicationEvent> appEventHandler = mock(EventHandler.class);
+ dispatcher.register(ApplicationEventType.class, appEventHandler);
+
+ LogAggregationService logAggregationService =
+ new LogAggregationService(dispatcher, this.context, delSrvc,
+ mockedDirSvc);
+ logAggregationService.init(this.conf);
+ logAggregationService.start();
+
+ ApplicationId application1 = BuilderUtils.newApplicationId(1234, 1);
+ logAggregationService.handle(new LogHandlerAppStartedEvent(
+ application1, this.user, null,
+ ContainerLogsRetentionPolicy.ALL_CONTAINERS, this.acls));
+
+ logAggregationService.stop();
+ }
}
