Repository: hadoop Updated Branches: refs/heads/trunk ae2177d29 -> 682ea21f2
Revert "YARN-7677. Docker image cannot set HADOOP_CONF_DIR. Contributed by Jim Brennan" This reverts commit 12eaae383ad06de8f9959241b2451dec82cf9ceb. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/682ea21f Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/682ea21f Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/682ea21f Branch: refs/heads/trunk Commit: 682ea21f2bbc587e1b727b3c895c2f513a908432 Parents: ae2177d Author: Jason Lowe <jl...@apache.org> Authored: Thu Feb 1 12:14:09 2018 -0600 Committer: Jason Lowe <jl...@apache.org> Committed: Thu Feb 1 12:14:09 2018 -0600 ---------------------------------------------------------------------- .../server/nodemanager/ContainerExecutor.java | 31 ++++++++++++-------- .../nodemanager/LinuxContainerExecutor.java | 8 +++++ .../launcher/ContainerLaunch.java | 15 ++-------- .../runtime/DefaultLinuxContainerRuntime.java | 6 ++++ .../DelegatingLinuxContainerRuntime.java | 11 +++++++ .../runtime/DockerLinuxContainerRuntime.java | 7 +++++ .../runtime/ContainerRuntime.java | 11 +++++++ .../launcher/TestContainerLaunch.java | 10 ++----- 8 files changed, 66 insertions(+), 33 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/682ea21f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java index 6241733..f4279a3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java @@ -346,6 +346,7 @@ public abstract class ContainerExecutor implements Configurable { public void writeLaunchEnv(OutputStream out, Map<String, String> environment, Map<Path, List<String>> resources, List<String> command, Path logDir, String user, String outFilename) throws IOException { + updateEnvForWhitelistVars(environment); ContainerLaunch.ShellScriptBuilder sb = ContainerLaunch.ShellScriptBuilder.create(); @@ -363,19 +364,6 @@ public abstract class ContainerExecutor implements Configurable { for (Map.Entry<String, String> env : environment.entrySet()) { sb.env(env.getKey(), env.getValue()); } - // Whitelist environment variables are treated specially. - // Only add them if they are not already defined in the environment. - // Add them using special syntax to prevent them from eclipsing - // variables that may be set explicitly in the container image (e.g, - // in a docker image) - for(String var : whitelistVars) { - if (!environment.containsKey(var)) { - String val = getNMEnvVar(var); - if (val != null) { - sb.whitelistedEnv(var, val); - } - } - } } if (resources != null) { @@ -675,6 +663,23 @@ public abstract class ContainerExecutor implements Configurable { } } + /** + * Propagate variables from the nodemanager's environment into the + * container's environment if unspecified by the container. + * @param env the environment to update + * @see org.apache.hadoop.yarn.conf.YarnConfiguration#NM_ENV_WHITELIST + */ + protected void updateEnvForWhitelistVars(Map<String, String> env) { + for(String var : whitelistVars) { + if (!env.containsKey(var)) { + String val = getNMEnvVar(var); + if (val != null) { + env.put(var, val); + } + } + } + } + @VisibleForTesting protected String getNMEnvVar(String varname) { return System.getenv(varname); http://git-wip-us.apache.org/repos/asf/hadoop/blob/682ea21f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java index f0b9797..d359f31 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java @@ -66,6 +66,7 @@ import java.net.InetSocketAddress; import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.Map; import java.util.regex.Pattern; import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.*; @@ -472,6 +473,13 @@ public class LinuxContainerExecutor extends ContainerExecutor { } @Override + protected void updateEnvForWhitelistVars(Map<String, String> env) { + if (linuxContainerRuntime.useWhitelistEnv(env)) { + super.updateEnvForWhitelistVars(env); + } + } + + @Override public int launchContainer(ContainerStartContext ctx) throws IOException, ConfigurationException { Container container = ctx.getContainer(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/682ea21f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java index 309a1d8..7f43458 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java @@ -1135,9 +1135,6 @@ public class ContainerLaunch implements Callable<Integer> { public abstract void env(String key, String value) throws IOException; - public abstract void whitelistedEnv(String key, String value) - throws IOException; - public abstract void echo(String echoStr) throws IOException; public final void symlink(Path src, Path dst) throws IOException { @@ -1258,11 +1255,6 @@ public class ContainerLaunch implements Callable<Integer> { } @Override - public void whitelistedEnv(String key, String value) throws IOException { - line("export ", key, "=${", key, ":-", "\"", value, "\"}"); - } - - @Override public void echo(final String echoStr) throws IOException { line("echo \"" + echoStr + "\""); } @@ -1353,11 +1345,6 @@ public class ContainerLaunch implements Callable<Integer> { } @Override - public void whitelistedEnv(String key, String value) throws IOException { - env(key, value); - } - - @Override public void echo(final String echoStr) throws IOException { lineWithLenCheck("@echo \"", echoStr, "\""); } @@ -1456,6 +1443,8 @@ public class ContainerLaunch implements Callable<Integer> { environment.put(Environment.PWD.name(), pwd.toString()); + putEnvIfAbsent(environment, Environment.HADOOP_CONF_DIR.name()); + if (!Shell.WINDOWS) { environment.put("JVM_PID", "$$"); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/682ea21f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DefaultLinuxContainerRuntime.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DefaultLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DefaultLinuxContainerRuntime.java index 83380ee..b50d56c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DefaultLinuxContainerRuntime.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DefaultLinuxContainerRuntime.java @@ -37,6 +37,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.List; +import java.util.Map; import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.*; @@ -73,6 +74,11 @@ public class DefaultLinuxContainerRuntime implements LinuxContainerRuntime { } @Override + public boolean useWhitelistEnv(Map<String, String> env) { + return true; + } + + @Override public void prepareContainer(ContainerRuntimeContext ctx) throws ContainerExecutionException { //nothing to do here at the moment. http://git-wip-us.apache.org/repos/asf/hadoop/blob/682ea21f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DelegatingLinuxContainerRuntime.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DelegatingLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DelegatingLinuxContainerRuntime.java index 675bffb..dd10617 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DelegatingLinuxContainerRuntime.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DelegatingLinuxContainerRuntime.java @@ -94,6 +94,17 @@ public class DelegatingLinuxContainerRuntime implements LinuxContainerRuntime { } } + @Override + public boolean useWhitelistEnv(Map<String, String> env) { + try { + LinuxContainerRuntime runtime = pickContainerRuntime(env); + return runtime.useWhitelistEnv(env); + } catch (ContainerExecutionException e) { + LOG.debug("Unable to determine runtime"); + return false; + } + } + @VisibleForTesting LinuxContainerRuntime pickContainerRuntime( Map<String, String> environment) throws ContainerExecutionException { http://git-wip-us.apache.org/repos/asf/hadoop/blob/682ea21f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java index 26af9a0..601c32c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java @@ -366,6 +366,13 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime { return capabilities; } + @Override + public boolean useWhitelistEnv(Map<String, String> env) { + // Avoid propagating nodemanager environment variables into the container + // so those variables can be picked up from the Docker image instead. + return false; + } + private String runDockerVolumeCommand(DockerVolumeCommand dockerVolumeCommand, Container container) throws ContainerExecutionException { try { http://git-wip-us.apache.org/repos/asf/hadoop/blob/682ea21f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/runtime/ContainerRuntime.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/runtime/ContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/runtime/ContainerRuntime.java index 7caa0ed..aa294fc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/runtime/ContainerRuntime.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/runtime/ContainerRuntime.java @@ -24,6 +24,8 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container; +import java.util.Map; + /** * An abstraction for various container runtime implementations. Examples * include Process Tree, Docker, Appc runtimes etc. These implementations @@ -83,4 +85,13 @@ public interface ContainerRuntime { * and hostname */ String[] getIpAndHost(Container container) throws ContainerExecutionException; + + /** + * Whether to propagate the whitelist of environment variables from the + * nodemanager environment into the container environment. + * @param env the container's environment variables + * @return true if whitelist variables should be propagated, false otherwise + * @see org.apache.hadoop.yarn.conf.YarnConfiguration#NM_ENV_WHITELIST + */ + boolean useWhitelistEnv(Map<String, String> env); } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/682ea21f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java index 5e57bb5..5923f8e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java @@ -337,8 +337,7 @@ public class TestContainerLaunch extends BaseContainerManagerTest { Assert.assertFalse(shellContent.contains("HADOOP_HDFS_HOME")); // Available in env and in whitelist Assert.assertTrue(shellContent.contains( - "export HADOOP_YARN_HOME=${HADOOP_YARN_HOME:-\"nodemanager_yarn_home\"}" - )); + "export HADOOP_YARN_HOME=\"nodemanager_yarn_home\"")); fos.flush(); fos.close(); } @@ -383,12 +382,9 @@ public class TestContainerLaunch extends BaseContainerManagerTest { // Whitelisted variable overridden by container Assert.assertTrue(shellContent.contains( "export HADOOP_MAPRED_HOME=\"/opt/hadoopbuild\"")); - // Available in env but not in whitelist + // Verify no whitelisted variables inherited from NM env Assert.assertFalse(shellContent.contains("HADOOP_HDFS_HOME")); - // Available in env and in whitelist - Assert.assertTrue(shellContent.contains( - "export HADOOP_YARN_HOME=${HADOOP_YARN_HOME:-\"nodemanager_yarn_home\"}" - )); + Assert.assertFalse(shellContent.contains("HADOOP_YARN_HOME")); fos.flush(); fos.close(); } --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
