Repository: hadoop
Updated Branches:
refs/heads/branch-2.6 5f05e5e5b -> 16e3dc24c
YARN-3925. ContainerLogsUtils#getContainerLogFile fails to read container log
files from full disks. Contributed by zhihai xu
(cherry picked from commit ff9c13e0a739bb13115167dc661b6a16b2ed2c04)
Conflicts:
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestContainerLogsPage.java
(cherry picked from commit e8410c0175518a7ca4f44921e950050854895563)
Conflicts:
hadoop-yarn-project/CHANGES.txt
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/271875a4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/271875a4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/271875a4
Branch: refs/heads/branch-2.6
Commit: 271875a42697af68617a5e4db100cc69e3934713
Parents: 5f05e5e
Author: Jason Lowe <[email protected]>
Authored: Mon Nov 23 20:57:57 2015 +0000
Committer: Jason Lowe <[email protected]>
Committed: Mon Nov 23 20:57:57 2015 +0000
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 ++
.../nodemanager/LocalDirsHandlerService.java | 35 +++++++++++++-
.../webapp/TestContainerLogsPage.java | 50 +++++++++++++++++++-
3 files changed, 85 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/271875a4/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index bbf0e9e..97e3b5b 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -30,6 +30,9 @@ Release 2.6.3 - UNRELEASED
YARN-4344. NMs reconnecting with changed capabilities can lead to wrong
cluster resource calculations (Varun Vasudev via jlowe)
+ YARN-3925. ContainerLogsUtils#getContainerLogFile fails to read container
+ log files from full disks. (zhihai xu via jlowe)
+
Release 2.6.2 - 2015-10-28
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/271875a4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LocalDirsHandlerService.java
----------------------------------------------------------------------
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LocalDirsHandlerService.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LocalDirsHandlerService.java
index d6950a3..07b1037 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LocalDirsHandlerService.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LocalDirsHandlerService.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.yarn.server.nodemanager;
+import java.io.File;
import java.io.IOException;
import java.net.URI;
import java.util.ArrayList;
@@ -31,6 +32,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileContext;
+import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocalDirAllocator;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
@@ -440,6 +442,35 @@ public class LocalDirsHandlerService extends
AbstractService {
return disksTurnedGood;
}
+ private Path getPathToRead(String pathStr, List<String> dirs)
+ throws IOException {
+ // remove the leading slash from the path (to make sure that the uri
+ // resolution results in a valid path on the dir being checked)
+ if (pathStr.startsWith("/")) {
+ pathStr = pathStr.substring(1);
+ }
+
+ FileSystem localFS = FileSystem.getLocal(getConfig());
+ for (String dir : dirs) {
+ try {
+ Path tmpDir = new Path(dir);
+ File tmpFile = tmpDir.isAbsolute()
+ ? new File(localFS.makeQualified(tmpDir).toUri())
+ : new File(dir);
+ Path file = new Path(tmpFile.getPath(), pathStr);
+ if (localFS.exists(file)) {
+ return file;
+ }
+ } catch (IOException ie) {
+ // ignore
+ LOG.warn("Failed to find " + pathStr + " at " + dir, ie);
+ }
+ }
+
+ throw new IOException("Could not find " + pathStr + " in any of" +
+ " the directories");
+ }
+
public Path getLocalPathForWrite(String pathStr) throws IOException {
return localDirsAllocator.getLocalPathForWrite(pathStr, getConfig());
}
@@ -457,9 +488,9 @@ public class LocalDirsHandlerService extends
AbstractService {
}
public Path getLogPathToRead(String pathStr) throws IOException {
- return logDirsAllocator.getLocalPathToRead(pathStr, getConfig());
+ return getPathToRead(pathStr, getLogDirsForRead());
}
-
+
public static String[] validatePaths(String[] paths) {
ArrayList<String> validPaths = new ArrayList<String>();
for (int i = 0; i < paths.length; ++i) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/271875a4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestContainerLogsPage.java
----------------------------------------------------------------------
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestContainerLogsPage.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestContainerLogsPage.java
index 39c52d9..979e745 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestContainerLogsPage.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestContainerLogsPage.java
@@ -39,6 +39,7 @@ import java.util.concurrent.ConcurrentMap;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.nativeio.NativeIO;
import org.apache.hadoop.security.UserGroupInformation;
@@ -62,6 +63,7 @@ import
org.apache.hadoop.yarn.server.nodemanager.NodeManager.NMContext;
import
org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application;
import
org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
import
org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerState;
+import
org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainerLaunch;
import
org.apache.hadoop.yarn.server.nodemanager.recovery.NMNullStateStoreService;
import
org.apache.hadoop.yarn.server.nodemanager.webapp.ContainerLogsPage.ContainersLogsBlock;
import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
@@ -136,7 +138,53 @@ public class TestContainerLogsPage {
File containerLogDir = new File(absLogDir, appId + "/" + container1);
Assert.assertTrue(dirs.contains(containerLogDir));
}
-
+
+ @Test(timeout=30000)
+ public void testContainerLogFile() throws IOException, YarnException {
+ File absLogDir = new File("target",
+ TestNMWebServer.class.getSimpleName() + "LogDir").getAbsoluteFile();
+ String logdirwithFile = absLogDir.toURI().toString();
+ Configuration conf = new Configuration();
+ conf.set(YarnConfiguration.NM_LOG_DIRS, logdirwithFile);
+ conf.setFloat(YarnConfiguration.NM_MAX_PER_DISK_UTILIZATION_PERCENTAGE,
+ 0.0f);
+ LocalDirsHandlerService dirsHandler = new LocalDirsHandlerService();
+ dirsHandler.init(conf);
+ NMContext nmContext = new NodeManager.NMContext(null, null, dirsHandler,
+ new ApplicationACLsManager(conf), new NMNullStateStoreService());
+ // Add an application and the corresponding containers
+ String user = "nobody";
+ long clusterTimeStamp = 1234;
+ ApplicationId appId = BuilderUtils.newApplicationId(
+ clusterTimeStamp, 1);
+ Application app = mock(Application.class);
+ when(app.getUser()).thenReturn(user);
+ when(app.getAppId()).thenReturn(appId);
+ ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(
+ appId, 1);
+ ContainerId containerId = BuilderUtils.newContainerId(
+ appAttemptId, 1);
+ nmContext.getApplications().put(appId, app);
+
+ MockContainer container =
+ new MockContainer(appAttemptId, new AsyncDispatcher(), conf, user,
+ appId, 1);
+ container.setState(ContainerState.RUNNING);
+ nmContext.getContainers().put(containerId, container);
+ File containerLogDir = new File(absLogDir,
+ ContainerLaunch.getRelativeContainerLogDir(appId.toString(),
+ containerId.toString()));
+ containerLogDir.mkdirs();
+ String fileName = "fileName";
+ File containerLogFile = new File(containerLogDir, fileName);
+ containerLogFile.createNewFile();
+ File file = ContainerLogsUtils.getContainerLogFile(containerId,
+ fileName, user, nmContext);
+ Assert.assertEquals(containerLogFile.toURI().toString(),
+ file.toURI().toString());
+ FileUtil.fullyDelete(absLogDir);
+ }
+
@Test(timeout = 10000)
public void testContainerLogPageAccess() throws IOException {
// SecureIOUtils require Native IO to be enabled. This test will run
