Repository: hadoop Updated Branches: refs/heads/trunk fedabcad4 -> d8923cdbf
YARN-7363. ContainerLocalizer don't have a valid log4j config in case of Linux container executor. (Contributed by Yufei Gu) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d8923cdb Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d8923cdb Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d8923cdb Branch: refs/heads/trunk Commit: d8923cdbf1567aee10a54f144fef734d1465ebed Parents: fedabca Author: Yufei Gu <yu...@apache.org> Authored: Mon Nov 27 11:47:11 2017 -0800 Committer: Yufei Gu <yu...@apache.org> Committed: Mon Nov 27 14:31:52 2017 -0800 ---------------------------------------------------------------------- .../hadoop/yarn/conf/YarnConfiguration.java | 6 +++ .../src/main/resources/yarn-default.xml | 8 ++++ .../nodemanager/LinuxContainerExecutor.java | 28 +++++++++++- .../WindowsSecureContainerExecutor.java | 2 +- .../localizer/ContainerLocalizer.java | 46 +++++++++++++++++++- .../TestLinuxContainerExecutorWithMocks.java | 19 +++++--- 6 files changed, 98 insertions(+), 11 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8923cdb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index ead9977..c1024ea 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -1675,6 +1675,12 @@ public class YarnConfiguration extends Configuration { public static final String NM_CONTAINER_LOCALIZER_JAVA_OPTS_DEFAULT = "-Xmx256m"; + /** The log level of container localizer process. */ + public static final String NM_CONTAINER_LOCALIZER_LOG_LEVEL= + NM_PREFIX + "container-localizer.log.level"; + public static final String NM_CONTAINER_LOCALIZER_LOG_LEVEL_DEFAULT = + "INFO"; + /** Prefix for runtime configuration constants. */ public static final String LINUX_CONTAINER_RUNTIME_PREFIX = NM_PREFIX + "runtime.linux."; http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8923cdb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 12cb902..dd9c6bd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -1165,6 +1165,14 @@ <property> <description> + The log level for container localizer while it is an independent process. + </description> + <name>yarn.nodemanager.container-localizer.log.level</name> + <value>INFO</value> + </property> + + <property> + <description> Where to store container logs. An application's localized log directory will be found in ${yarn.nodemanager.log-dirs}/application_${appid}. Individual containers' log directories will be below this, in directories http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8923cdb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java index e8c46a2..eaf664f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java @@ -33,6 +33,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.ConfigurationException; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerDiagnosticsUpdateEvent; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainerLaunch; import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperation; import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperationException; import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperationExecutor; @@ -382,6 +383,10 @@ public class LinuxContainerExecutor extends ContainerExecutor { List<String> localizerArgs = new ArrayList<>(); buildMainArgs(localizerArgs, user, appId, locId, nmAddr, localDirs); + + Path containerLogDir = getContainerLogDir(dirsHandler, appId, locId); + localizerArgs = replaceWithContainerLogDir(localizerArgs, containerLogDir); + initializeContainerOp.appendArgs(localizerArgs); try { @@ -402,6 +407,27 @@ public class LinuxContainerExecutor extends ContainerExecutor { } } + private List<String> replaceWithContainerLogDir(List<String> commands, + Path containerLogDir) { + List<String> newCmds = new ArrayList<>(commands.size()); + + for (String item : commands) { + newCmds.add(item.replace(ApplicationConstants.LOG_DIR_EXPANSION_VAR, + containerLogDir.toString())); + } + + return newCmds; + } + + private Path getContainerLogDir(LocalDirsHandlerService dirsHandler, + String appId, String containerId) throws IOException { + String relativeContainerLogDir = ContainerLaunch + .getRelativeContainerLogDir(appId, containerId); + + return dirsHandler.getLogPathForWrite(relativeContainerLogDir, + false); + } + /** * Set up the {@link ContainerLocalizer}. * @@ -417,7 +443,7 @@ public class LinuxContainerExecutor extends ContainerExecutor { public void buildMainArgs(List<String> command, String user, String appId, String locId, InetSocketAddress nmAddr, List<String> localDirs) { ContainerLocalizer.buildMainArgs(command, user, appId, locId, nmAddr, - localDirs); + localDirs, super.getConf()); } @Override http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8923cdb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/WindowsSecureContainerExecutor.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/WindowsSecureContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/WindowsSecureContainerExecutor.java index 7c124c3..e934472 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/WindowsSecureContainerExecutor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/WindowsSecureContainerExecutor.java @@ -702,7 +702,7 @@ public class WindowsSecureContainerExecutor extends DefaultContainerExecutor { command.addAll(ContainerLocalizer.getJavaOpts(getConf())); ContainerLocalizer.buildMainArgs(command, user, appId, locId, nmAddr, - localDirs); + localDirs, super.getConf()); String cmdLine = StringUtils.join(command, " "); http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8923cdb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java index 2378c45..541911a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java @@ -63,6 +63,8 @@ import org.apache.hadoop.util.DiskValidatorFactory; import org.apache.hadoop.util.Shell; import org.apache.hadoop.util.concurrent.HadoopExecutors; import org.apache.hadoop.yarn.YarnUncaughtExceptionHandler; +import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; +import org.apache.hadoop.yarn.api.ApplicationConstants; import org.apache.hadoop.yarn.api.records.LocalResource; import org.apache.hadoop.yarn.api.records.LocalResourceVisibility; import org.apache.hadoop.yarn.api.records.SerializedException; @@ -408,8 +410,12 @@ public class ContainerLocalizer { */ public static void buildMainArgs(List<String> command, String user, String appId, String locId, - InetSocketAddress nmAddr, List<String> localDirs) { - + InetSocketAddress nmAddr, List<String> localDirs, Configuration conf) { + + String logLevel = conf.get(YarnConfiguration. + NM_CONTAINER_LOCALIZER_LOG_LEVEL, + YarnConfiguration.NM_CONTAINER_LOCALIZER_LOG_LEVEL_DEFAULT); + addLog4jSystemProperties(logLevel, command); command.add(ContainerLocalizer.class.getName()); command.add(user); command.add(appId); @@ -421,6 +427,16 @@ public class ContainerLocalizer { } } + private static void addLog4jSystemProperties( + String logLevel, List<String> command) { + command.add("-Dlog4j.configuration=container-log4j.properties"); + command.add("-D" + YarnConfiguration.YARN_APP_CONTAINER_LOG_DIR + "=" + + ApplicationConstants.LOG_DIR_EXPANSION_VAR); + command.add("-D" + YarnConfiguration.YARN_APP_CONTAINER_LOG_SIZE + "=0"); + command.add("-Dhadoop.root.logger=" + logLevel + ",CLA"); + command.add("-Dhadoop.root.logfile=container-localizer-syslog"); + } + public static void main(String[] argv) throws Throwable { Thread.setDefaultUncaughtExceptionHandler(new YarnUncaughtExceptionHandler()); int nRet = 0; @@ -431,6 +447,7 @@ public class ContainerLocalizer { // MKDIR $x/$user/appcache/$appid/filecache // LOAD $x/$user/appcache/$appid/appTokens try { + createLogDir(); String user = argv[0]; String appId = argv[1]; String locId = argv[2]; @@ -466,6 +483,31 @@ public class ContainerLocalizer { } } + /** + * Create the log directory, if the directory exists, make sure its permission + * is 750. + */ + private static void createLogDir() { + FileContext localFs; + try { + localFs = FileContext.getLocalFSFileContext(new Configuration()); + + String logDir = System.getProperty( + YarnConfiguration.YARN_APP_CONTAINER_LOG_DIR); + + if (logDir != null && !logDir.trim().isEmpty()) { + Path containerLogPath = new Path(logDir); + FsPermission containerLogDirPerm= new FsPermission((short)0750); + localFs.mkdir(containerLogPath, containerLogDirPerm, true); + // set permission again to make sure the permission is correct + // in case the directory is already there. + localFs.setPermission(containerLogPath, containerLogDirPerm); + } + } catch (IOException e) { + throw new YarnRuntimeException("Unable to create the log dir", e); + } + } + private static void initDirs(Configuration conf, String user, String appId, FileContext lfs, List<Path> localDirs) throws IOException { if (null == localDirs || 0 == localDirs.size()) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8923cdb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java index e1d9c69..9a56656 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java @@ -269,7 +269,7 @@ public class TestLinuxContainerExecutorWithMocks { .build()); List<String> result=readMockParams(); - Assert.assertEquals(result.size(), 19); + Assert.assertEquals(result.size(), 24); Assert.assertEquals(result.get(0), YarnConfiguration.DEFAULT_NM_NONSECURE_MODE_LOCAL_USER); Assert.assertEquals(result.get(1), "test"); Assert.assertEquals(result.get(2), "0" ); @@ -277,12 +277,17 @@ public class TestLinuxContainerExecutorWithMocks { Assert.assertEquals(result.get(4), "/bin/nmPrivateCTokensPath"); Assert.assertEquals(result.get(8), "-classpath" ); Assert.assertEquals(result.get(11), "-Xmx256m" ); - Assert.assertEquals(result.get(12),"org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer" ); - Assert.assertEquals(result.get(13), "test"); - Assert.assertEquals(result.get(14), "application_0"); - Assert.assertEquals(result.get(15),"12345" ); - Assert.assertEquals(result.get(16),"localhost" ); - Assert.assertEquals(result.get(17),"8040" ); + Assert.assertEquals(result.get(12), "-Dlog4j.configuration=container-log4j.properties" ); + Assert.assertEquals(result.get(13), "-Dyarn.app.container.log.dir=${yarn.log.dir}/userlogs/application_0/12345"); + Assert.assertEquals(result.get(14), "-Dyarn.app.container.log.filesize=0"); + Assert.assertEquals(result.get(15), "-Dhadoop.root.logger=INFO,CLA"); + Assert.assertEquals(result.get(16), "-Dhadoop.root.logfile=container-localizer-syslog"); + Assert.assertEquals(result.get(17),"org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer" ); + Assert.assertEquals(result.get(18), "test"); + Assert.assertEquals(result.get(19), "application_0"); + Assert.assertEquals(result.get(20),"12345" ); + Assert.assertEquals(result.get(21),"localhost" ); + Assert.assertEquals(result.get(22),"8040" ); } catch (InterruptedException e) { LOG.error("Error:"+e.getMessage(),e); --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
