Author: jianhe
Date: Mon Mar 17 21:54:25 2014
New Revision: 1578619
URL: http://svn.apache.org/r1578619
Log:
Merge r1578618 from branch-2. Fixed AM container log to show on NM web page
after application finishes if log-aggregation is disabled. Contributed by
Rohith Sharmaks
Modified:
hadoop/common/branches/branch-2.4/hadoop-yarn-project/CHANGES.txt
hadoop/common/branches/branch-2.4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerLogsUtils.java
hadoop/common/branches/branch-2.4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestContainerLogsPage.java
hadoop/common/branches/branch-2.4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java
Modified: hadoop/common/branches/branch-2.4/hadoop-yarn-project/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.4/hadoop-yarn-project/CHANGES.txt?rev=1578619&r1=1578618&r2=1578619&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.4/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/branches/branch-2.4/hadoop-yarn-project/CHANGES.txt Mon Mar
17 21:54:25 2014
@@ -442,6 +442,9 @@ Release 2.4.0 - UNRELEASED
YARN-1685. Fixed few bugs related to handling of containers' log-URLs on
ResourceManager and history-service. (Zhijie Shen via vinodkv)
+ YARN-1206. Fixed AM container log to show on NM web page after application
+ finishes if log-aggregation is disabled. (Rohith Sharmaks via jianhe)
+
Release 2.3.1 - UNRELEASED
INCOMPATIBLE CHANGES
Modified:
hadoop/common/branches/branch-2.4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerLogsUtils.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerLogsUtils.java?rev=1578619&r1=1578618&r2=1578619&view=diff
==============================================================================
---
hadoop/common/branches/branch-2.4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerLogsUtils.java
(original)
+++
hadoop/common/branches/branch-2.4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerLogsUtils.java
Mon Mar 17 21:54:25 2014
@@ -56,13 +56,18 @@ public class ContainerLogsUtils {
public static List<File> getContainerLogDirs(ContainerId containerId,
String remoteUser, Context context) throws YarnException {
Container container = context.getContainers().get(containerId);
- if (container == null) {
- throw new YarnException("Container does not exist.");
- }
Application application = getApplicationForContainer(containerId, context);
checkAccess(remoteUser, application, context);
- checkState(container.getContainerState());
+ // It is not required to have null check for container ( container == null
)
+ // and throw back exception.Because when container is completed,
NodeManager
+ // remove container information from its NMContext.Configuring log
+ // aggregation to false, container log view request is forwarded to NM. NM
+ // does not have completed container information,but still NM serve
request for
+ // reading container logs.
+ if (container != null) {
+ checkState(container.getContainerState());
+ }
return getContainerLogDirs(containerId, context.getLocalDirsHandler());
}
@@ -91,14 +96,12 @@ public class ContainerLogsUtils {
public static File getContainerLogFile(ContainerId containerId,
String fileName, String remoteUser, Context context) throws
YarnException {
Container container = context.getContainers().get(containerId);
- if (container == null) {
- throw new NotFoundException("Container with id " + containerId
- + " not found.");
- }
Application application = getApplicationForContainer(containerId, context);
checkAccess(remoteUser, application, context);
- checkState(container.getContainerState());
+ if (container != null) {
+ checkState(container.getContainerState());
+ }
try {
LocalDirsHandlerService dirsHandler = context.getLocalDirsHandler();
Modified:
hadoop/common/branches/branch-2.4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestContainerLogsPage.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestContainerLogsPage.java?rev=1578619&r1=1578618&r2=1578619&view=diff
==============================================================================
---
hadoop/common/branches/branch-2.4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestContainerLogsPage.java
(original)
+++
hadoop/common/branches/branch-2.4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestContainerLogsPage.java
Mon Mar 17 21:54:25 2014
@@ -49,10 +49,13 @@ import org.apache.hadoop.yarn.factory.pr
import org.apache.hadoop.yarn.server.nodemanager.Context;
import org.apache.hadoop.yarn.server.nodemanager.LocalDirsHandlerService;
import org.apache.hadoop.yarn.server.nodemanager.NodeHealthCheckerService;
+import org.apache.hadoop.yarn.server.nodemanager.NodeManager;
+import org.apache.hadoop.yarn.server.nodemanager.NodeManager.NMContext;
import
org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application;
import
org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
import
org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerState;
import
org.apache.hadoop.yarn.server.nodemanager.webapp.ContainerLogsPage.ContainersLogsBlock;
+import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.apache.hadoop.yarn.webapp.YarnWebParams;
import org.apache.hadoop.yarn.webapp.test.WebAppTests;
@@ -74,6 +77,7 @@ public class TestContainerLogsPage {
NodeHealthCheckerService healthChecker = new NodeHealthCheckerService();
healthChecker.init(conf);
LocalDirsHandlerService dirsHandler = healthChecker.getDiskHandler();
+ NMContext nmContext = new NodeManager.NMContext(null, null, dirsHandler,
new ApplicationACLsManager(conf));
// Add an application and the corresponding containers
RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(conf);
String user = "nobody";
@@ -87,9 +91,21 @@ public class TestContainerLogsPage {
appId, 1);
ContainerId container1 = BuilderUtils.newContainerId(recordFactory, appId,
appAttemptId, 0);
+ nmContext.getApplications().put(appId, app);
+
+ MockContainer container =
+ new MockContainer(appAttemptId, new AsyncDispatcher(), conf, user,
+ appId, 1);
+ container.setState(ContainerState.RUNNING);
+ nmContext.getContainers().put(container1, container);
List<File> files = null;
- files = ContainerLogsUtils.getContainerLogDirs(
- container1, dirsHandler);
+ files = ContainerLogsUtils.getContainerLogDirs(container1, user,
nmContext);
+ Assert.assertTrue(!(files.get(0).toString().contains("file:")));
+
+ // After container is completed, it is removed from nmContext
+ nmContext.getContainers().remove(container1);
+ Assert.assertNull(nmContext.getContainers().get(container1));
+ files = ContainerLogsUtils.getContainerLogDirs(container1, user,
nmContext);
Assert.assertTrue(!(files.get(0).toString().contains("file:")));
}
Modified:
hadoop/common/branches/branch-2.4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java?rev=1578619&r1=1578618&r2=1578619&view=diff
==============================================================================
---
hadoop/common/branches/branch-2.4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java
(original)
+++
hadoop/common/branches/branch-2.4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java
Mon Mar 17 21:54:25 2014
@@ -352,6 +352,16 @@ public class TestNMWebServices extends J
Assert.assertEquals(Status.NOT_FOUND.getStatusCode(),
response.getStatus());
responseText = response.getEntity(String.class);
assertTrue(responseText.contains("Cannot find this log on the local
disk."));
+
+ // After container is completed, it is removed from nmContext
+ nmContext.getContainers().remove(containerId);
+ Assert.assertNull(nmContext.getContainers().get(containerId));
+ response =
+ r.path("ws").path("v1").path("node").path("containerlogs")
+ .path(containerIdStr).path(filename).accept(MediaType.TEXT_PLAIN)
+ .get(ClientResponse.class);
+ responseText = response.getEntity(String.class);
+ assertEquals(logMessage, responseText);
}
public void verifyNodesXML(NodeList nodes) throws JSONException, Exception {
