Repository: hadoop
Updated Branches:
refs/heads/branch-2 45f59bde6 -> de144175a
YARN-7363. ContainerLocalizer don't have a valid log4j config in case of Linux
container executor. (Contributed by Yufei Gu)
(cherry picked from commit d8923cdbf1567aee10a54f144fef734d1465ebed)
(cherry picked from commit c0ab3c8407b110dd9083d775e5ea823cbf73e7d4)
Conflicts:
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/de144175
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/de144175
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/de144175
Branch: refs/heads/branch-2
Commit: de144175a4dc82ddce2a0b48f97f05101514e616
Parents: 45f59bd
Author: Yufei Gu <[email protected]>
Authored: Mon Nov 27 11:47:11 2017 -0800
Committer: Yufei Gu <[email protected]>
Committed: Mon Nov 27 14:54:27 2017 -0800
----------------------------------------------------------------------
.../hadoop/yarn/conf/YarnConfiguration.java | 6 +++
.../src/main/resources/yarn-default.xml | 8 ++++
.../nodemanager/LinuxContainerExecutor.java | 28 +++++++++++-
.../WindowsSecureContainerExecutor.java | 2 +-
.../localizer/ContainerLocalizer.java | 46 +++++++++++++++++++-
.../TestLinuxContainerExecutorWithMocks.java | 19 +++++---
6 files changed, 98 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de144175/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index edeec9f..4820903 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -1540,6 +1540,12 @@ public class YarnConfiguration extends Configuration {
public static final String NM_CONTAINER_LOCALIZER_JAVA_OPTS_DEFAULT =
"-Xmx256m";
+ /** The log level of container localizer process. */
+ public static final String NM_CONTAINER_LOCALIZER_LOG_LEVEL=
+ NM_PREFIX + "container-localizer.log.level";
+ public static final String NM_CONTAINER_LOCALIZER_LOG_LEVEL_DEFAULT =
+ "INFO";
+
/** The Docker image name(For DockerContainerExecutor).*/
public static final String NM_DOCKER_CONTAINER_EXECUTOR_IMAGE_NAME =
NM_PREFIX + "docker-container-executor.image-name";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de144175/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
----------------------------------------------------------------------
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 71dd72a..e869e30 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -1152,6 +1152,14 @@
<property>
<description>
+ The log level for container localizer while it is an independent process.
+ </description>
+ <name>yarn.nodemanager.container-localizer.log.level</name>
+ <value>INFO</value>
+ </property>
+
+ <property>
+ <description>
Where to store container logs. An application's localized log directory
will be found in ${yarn.nodemanager.log-dirs}/application_${appid}.
Individual containers' log directories will be below this, in
directories
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de144175/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
----------------------------------------------------------------------
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
index c1c3dc0..a75a32c 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
@@ -33,6 +33,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.ConfigurationException;
import
org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
import
org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerDiagnosticsUpdateEvent;
+import
org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainerLaunch;
import
org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperation;
import
org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperationException;
import
org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperationExecutor;
@@ -381,6 +382,10 @@ public class LinuxContainerExecutor extends
ContainerExecutor {
List<String> localizerArgs = new ArrayList<>();
buildMainArgs(localizerArgs, user, appId, locId, nmAddr, localDirs);
+
+ Path containerLogDir = getContainerLogDir(dirsHandler, appId, locId);
+ localizerArgs = replaceWithContainerLogDir(localizerArgs, containerLogDir);
+
initializeContainerOp.appendArgs(localizerArgs);
try {
@@ -401,6 +406,27 @@ public class LinuxContainerExecutor extends
ContainerExecutor {
}
}
+ private List<String> replaceWithContainerLogDir(List<String> commands,
+ Path containerLogDir) {
+ List<String> newCmds = new ArrayList<>(commands.size());
+
+ for (String item : commands) {
+ newCmds.add(item.replace(ApplicationConstants.LOG_DIR_EXPANSION_VAR,
+ containerLogDir.toString()));
+ }
+
+ return newCmds;
+ }
+
+ private Path getContainerLogDir(LocalDirsHandlerService dirsHandler,
+ String appId, String containerId) throws IOException {
+ String relativeContainerLogDir = ContainerLaunch
+ .getRelativeContainerLogDir(appId, containerId);
+
+ return dirsHandler.getLogPathForWrite(relativeContainerLogDir,
+ false);
+ }
+
/**
* Set up the {@link ContainerLocalizer}.
*
@@ -416,7 +442,7 @@ public class LinuxContainerExecutor extends
ContainerExecutor {
public void buildMainArgs(List<String> command, String user, String appId,
String locId, InetSocketAddress nmAddr, List<String> localDirs) {
ContainerLocalizer.buildMainArgs(command, user, appId, locId, nmAddr,
- localDirs);
+ localDirs, super.getConf());
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de144175/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/WindowsSecureContainerExecutor.java
----------------------------------------------------------------------
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/WindowsSecureContainerExecutor.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/WindowsSecureContainerExecutor.java
index 7c124c3..e934472 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/WindowsSecureContainerExecutor.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/WindowsSecureContainerExecutor.java
@@ -702,7 +702,7 @@ public class WindowsSecureContainerExecutor extends
DefaultContainerExecutor {
command.addAll(ContainerLocalizer.getJavaOpts(getConf()));
ContainerLocalizer.buildMainArgs(command, user, appId, locId, nmAddr,
- localDirs);
+ localDirs, super.getConf());
String cmdLine = StringUtils.join(command, " ");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de144175/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java
----------------------------------------------------------------------
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java
index 8c63479..6e1ace4 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java
@@ -60,6 +60,8 @@ import org.apache.hadoop.util.DiskValidatorFactory;
import org.apache.hadoop.util.Shell;
import org.apache.hadoop.util.concurrent.HadoopExecutors;
import org.apache.hadoop.yarn.YarnUncaughtExceptionHandler;
+import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
+import org.apache.hadoop.yarn.api.ApplicationConstants;
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
import org.apache.hadoop.yarn.api.records.SerializedException;
@@ -407,8 +409,12 @@ public class ContainerLocalizer {
*/
public static void buildMainArgs(List<String> command,
String user, String appId, String locId,
- InetSocketAddress nmAddr, List<String> localDirs) {
-
+ InetSocketAddress nmAddr, List<String> localDirs, Configuration conf) {
+
+ String logLevel = conf.get(YarnConfiguration.
+ NM_CONTAINER_LOCALIZER_LOG_LEVEL,
+ YarnConfiguration.NM_CONTAINER_LOCALIZER_LOG_LEVEL_DEFAULT);
+ addLog4jSystemProperties(logLevel, command);
command.add(ContainerLocalizer.class.getName());
command.add(user);
command.add(appId);
@@ -420,6 +426,16 @@ public class ContainerLocalizer {
}
}
+ private static void addLog4jSystemProperties(
+ String logLevel, List<String> command) {
+ command.add("-Dlog4j.configuration=container-log4j.properties");
+ command.add("-D" + YarnConfiguration.YARN_APP_CONTAINER_LOG_DIR + "=" +
+ ApplicationConstants.LOG_DIR_EXPANSION_VAR);
+ command.add("-D" + YarnConfiguration.YARN_APP_CONTAINER_LOG_SIZE + "=0");
+ command.add("-Dhadoop.root.logger=" + logLevel + ",CLA");
+ command.add("-Dhadoop.root.logfile=container-localizer-syslog");
+ }
+
public static void main(String[] argv) throws Throwable {
Thread.setDefaultUncaughtExceptionHandler(new
YarnUncaughtExceptionHandler());
int nRet = 0;
@@ -430,6 +446,7 @@ public class ContainerLocalizer {
// MKDIR $x/$user/appcache/$appid/filecache
// LOAD $x/$user/appcache/$appid/appTokens
try {
+ createLogDir();
String user = argv[0];
String appId = argv[1];
String locId = argv[2];
@@ -465,6 +482,31 @@ public class ContainerLocalizer {
}
}
+ /**
+ * Create the log directory, if the directory exists, make sure its
permission
+ * is 750.
+ */
+ private static void createLogDir() {
+ FileContext localFs;
+ try {
+ localFs = FileContext.getLocalFSFileContext(new Configuration());
+
+ String logDir = System.getProperty(
+ YarnConfiguration.YARN_APP_CONTAINER_LOG_DIR);
+
+ if (logDir != null && !logDir.trim().isEmpty()) {
+ Path containerLogPath = new Path(logDir);
+ FsPermission containerLogDirPerm= new FsPermission((short)0750);
+ localFs.mkdir(containerLogPath, containerLogDirPerm, true);
+ // set permission again to make sure the permission is correct
+ // in case the directory is already there.
+ localFs.setPermission(containerLogPath, containerLogDirPerm);
+ }
+ } catch (IOException e) {
+ throw new YarnRuntimeException("Unable to create the log dir", e);
+ }
+ }
+
private static void initDirs(Configuration conf, String user, String appId,
FileContext lfs, List<Path> localDirs) throws IOException {
if (null == localDirs || 0 == localDirs.size()) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de144175/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java
----------------------------------------------------------------------
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java
index 6eb1d9d..db36868 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java
@@ -269,7 +269,7 @@ public class TestLinuxContainerExecutorWithMocks {
.build());
List<String> result=readMockParams();
- Assert.assertEquals(result.size(), 19);
+ Assert.assertEquals(result.size(), 24);
Assert.assertEquals(result.get(0),
YarnConfiguration.DEFAULT_NM_NONSECURE_MODE_LOCAL_USER);
Assert.assertEquals(result.get(1), "test");
Assert.assertEquals(result.get(2), "0" );
@@ -277,12 +277,17 @@ public class TestLinuxContainerExecutorWithMocks {
Assert.assertEquals(result.get(4), "/bin/nmPrivateCTokensPath");
Assert.assertEquals(result.get(8), "-classpath" );
Assert.assertEquals(result.get(11), "-Xmx256m" );
-
Assert.assertEquals(result.get(12),"org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer"
);
- Assert.assertEquals(result.get(13), "test");
- Assert.assertEquals(result.get(14), "application_0");
- Assert.assertEquals(result.get(15),"12345" );
- Assert.assertEquals(result.get(16),"localhost" );
- Assert.assertEquals(result.get(17),"8040" );
+ Assert.assertEquals(result.get(12),
"-Dlog4j.configuration=container-log4j.properties" );
+ Assert.assertEquals(result.get(13),
"-Dyarn.app.container.log.dir=${yarn.log.dir}/userlogs/application_0/12345");
+ Assert.assertEquals(result.get(14),
"-Dyarn.app.container.log.filesize=0");
+ Assert.assertEquals(result.get(15), "-Dhadoop.root.logger=INFO,CLA");
+ Assert.assertEquals(result.get(16),
"-Dhadoop.root.logfile=container-localizer-syslog");
+
Assert.assertEquals(result.get(17),"org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer"
);
+ Assert.assertEquals(result.get(18), "test");
+ Assert.assertEquals(result.get(19), "application_0");
+ Assert.assertEquals(result.get(20),"12345" );
+ Assert.assertEquals(result.get(21),"localhost" );
+ Assert.assertEquals(result.get(22),"8040" );
} catch (InterruptedException e) {
LOG.error("Error:"+e.getMessage(),e);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
