YARN-1912. ResourceLocalizer started without any jvm memory control. Contributed by Masatake Iwasaki
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6471d18b Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6471d18b Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6471d18b Branch: refs/heads/HDFS-7240 Commit: 6471d18bc72bc6c83ce31a03b5c5f5737847bb6d Parents: 333f9a8 Author: Xuan <[email protected]> Authored: Fri May 8 20:01:21 2015 -0700 Committer: Xuan <[email protected]> Committed: Fri May 8 20:01:21 2015 -0700 ---------------------------------------------------------------------- hadoop-yarn-project/CHANGES.txt | 3 +++ .../apache/hadoop/yarn/conf/YarnConfiguration.java | 7 +++++++ .../server/nodemanager/LinuxContainerExecutor.java | 1 + .../nodemanager/WindowsSecureContainerExecutor.java | 3 ++- .../localizer/ContainerLocalizer.java | 13 ++++++++++++- .../TestLinuxContainerExecutorWithMocks.java | 15 ++++++++------- 6 files changed, 33 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/6471d18b/hadoop-yarn-project/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 05ae144..8f07b79 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -215,6 +215,9 @@ Release 2.8.0 - UNRELEASED YARN-3271. FairScheduler: Move tests related to max-runnable-apps from TestFairScheduler to TestAppRunnability. (nijel via kasha) + YARN-1912. ResourceLocalizer started without any jvm memory control. + (Masatake Iwasaki via xgong) + OPTIMIZATIONS YARN-3339. TestDockerContainerExecutor should pull a single image and not http://git-wip-us.apache.org/repos/asf/hadoop/blob/6471d18b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 0851f3c..48a75c0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -1004,6 +1004,13 @@ public class YarnConfiguration extends Configuration { public static final String NM_HEALTH_CHECK_SCRIPT_OPTS = NM_PREFIX + "health-checker.script.opts"; + /** The JVM options used on forking ContainerLocalizer process + by container executor. */ + public static final String NM_CONTAINER_LOCALIZER_JAVA_OPTS_KEY = + NM_PREFIX + "container-localizer.java.opts"; + public static final String NM_CONTAINER_LOCALIZER_JAVA_OPTS_DEFAULT = + "-Xmx256m"; + /** The Docker image name(For DockerContainerExecutor).*/ public static final String NM_DOCKER_CONTAINER_EXECUTOR_IMAGE_NAME = NM_PREFIX + "docker-container-executor.image-name"; http://git-wip-us.apache.org/repos/asf/hadoop/blob/6471d18b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java index f8da958..16068d7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java @@ -244,6 +244,7 @@ public class LinuxContainerExecutor extends ContainerExecutor { if (javaLibPath != null) { command.add("-Djava.library.path=" + javaLibPath); } + command.addAll(ContainerLocalizer.getJavaOpts(getConf())); buildMainArgs(command, user, appId, locId, nmAddr, localDirs); String[] commandArray = command.toArray(new String[command.size()]); ShellCommandExecutor shExec = new ShellCommandExecutor(commandArray); http://git-wip-us.apache.org/repos/asf/hadoop/blob/6471d18b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/WindowsSecureContainerExecutor.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/WindowsSecureContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/WindowsSecureContainerExecutor.java index b7bec5f..619b845 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/WindowsSecureContainerExecutor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/WindowsSecureContainerExecutor.java @@ -697,7 +697,8 @@ public class WindowsSecureContainerExecutor extends DefaultContainerExecutor { if (javaLibPath != null) { command.add("-Djava.library.path=" + javaLibPath); } - + command.addAll(ContainerLocalizer.getJavaOpts(getConf())); + ContainerLocalizer.buildMainArgs(command, user, appId, locId, nmAddr, localDirs); http://git-wip-us.apache.org/repos/asf/hadoop/blob/6471d18b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java index 32e3553..f82f894 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java @@ -55,6 +55,7 @@ import org.apache.hadoop.util.DiskChecker; import org.apache.hadoop.yarn.YarnUncaughtExceptionHandler; import org.apache.hadoop.yarn.api.records.LocalResource; import org.apache.hadoop.yarn.api.records.SerializedException; +import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; @@ -312,7 +313,17 @@ public class ContainerLocalizer { status.addAllResources(currentResources); return status; } - + + /** + * Returns the JVM options to to launch the resource localizer. + * @param conf the configuration properties to launch the resource localizer. + */ + public static List<String> getJavaOpts(Configuration conf) { + String opts = conf.get(YarnConfiguration.NM_CONTAINER_LOCALIZER_JAVA_OPTS_KEY, + YarnConfiguration.NM_CONTAINER_LOCALIZER_JAVA_OPTS_DEFAULT); + return Arrays.asList(opts.split(" ")); + } + /** * Adds the ContainerLocalizer arguments for a @{link ShellCommandExecutor}, * as expected by ContainerLocalizer.main http://git-wip-us.apache.org/repos/asf/hadoop/blob/6471d18b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java index 98ab8e0..dce2cd3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java @@ -187,19 +187,20 @@ public class TestLinuxContainerExecutorWithMocks { try { mockExec.startLocalizer(nmPrivateCTokensPath, address, "test", "application_0", "12345", dirsHandler); List<String> result=readMockParams(); - Assert.assertEquals(result.size(), 17); + Assert.assertEquals(result.size(), 18); Assert.assertEquals(result.get(0), YarnConfiguration.DEFAULT_NM_NONSECURE_MODE_LOCAL_USER); Assert.assertEquals(result.get(1), "test"); Assert.assertEquals(result.get(2), "0" ); Assert.assertEquals(result.get(3),"application_0" ); Assert.assertEquals(result.get(4), "/bin/nmPrivateCTokensPath"); Assert.assertEquals(result.get(8), "-classpath" ); - Assert.assertEquals(result.get(11),"org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer" ); - Assert.assertEquals(result.get(12), "test"); - Assert.assertEquals(result.get(13), "application_0"); - Assert.assertEquals(result.get(14),"12345" ); - Assert.assertEquals(result.get(15),"localhost" ); - Assert.assertEquals(result.get(16),"8040" ); + Assert.assertEquals(result.get(11), "-Xmx256m" ); + Assert.assertEquals(result.get(12),"org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer" ); + Assert.assertEquals(result.get(13), "test"); + Assert.assertEquals(result.get(14), "application_0"); + Assert.assertEquals(result.get(15),"12345" ); + Assert.assertEquals(result.get(16),"localhost" ); + Assert.assertEquals(result.get(17),"8040" ); } catch (InterruptedException e) { LOG.error("Error:"+e.getMessage(),e);
