YARN-6472. Improve Java sandbox regex (gphillips via rkanter)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7b05bddb Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7b05bddb Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7b05bddb Branch: refs/heads/HDFS-10467 Commit: 7b05bddb7695f9e9d0070e035034582d9f57c45a Parents: 55b59b8 Author: Robert Kanter <[email protected]> Authored: Fri Apr 28 11:01:50 2017 -0700 Committer: Inigo <[email protected]> Committed: Tue May 2 14:52:05 2017 -0700 ---------------------------------------------------------------------- .../DelegatingLinuxContainerRuntime.java | 9 +++++---- .../JavaSandboxLinuxContainerRuntime.java | 13 ++++++++----- .../TestJavaSandboxLinuxContainerRuntime.java | 20 +++++++++++++++++--- 3 files changed, 30 insertions(+), 12 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b05bddb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DelegatingLinuxContainerRuntime.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DelegatingLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DelegatingLinuxContainerRuntime.java index 896ffd0..90b13a2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DelegatingLinuxContainerRuntime.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DelegatingLinuxContainerRuntime.java @@ -70,11 +70,12 @@ public class DelegatingLinuxContainerRuntime implements LinuxContainerRuntime { private LinuxContainerRuntime pickContainerRuntime( Map<String, String> environment){ LinuxContainerRuntime runtime; - - if (DockerLinuxContainerRuntime.isDockerContainerRequested(environment)){ + //Sandbox checked first to ensure DockerRuntime doesn't circumvent controls + if (javaSandboxLinuxContainerRuntime.isSandboxContainerRequested()){ + runtime = javaSandboxLinuxContainerRuntime; + } else if (DockerLinuxContainerRuntime + .isDockerContainerRequested(environment)){ runtime = dockerLinuxContainerRuntime; - } else if (javaSandboxLinuxContainerRuntime.isSandboxContainerRequested()) { - runtime = javaSandboxLinuxContainerRuntime; } else { runtime = defaultLinuxContainerRuntime; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b05bddb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/JavaSandboxLinuxContainerRuntime.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/JavaSandboxLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/JavaSandboxLinuxContainerRuntime.java index 6dc627b..1e5bf57 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/JavaSandboxLinuxContainerRuntime.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/JavaSandboxLinuxContainerRuntime.java @@ -141,8 +141,6 @@ public class JavaSandboxLinuxContainerRuntime this.configuration.get(YarnConfiguration.YARN_CONTAINER_SANDBOX, YarnConfiguration.DEFAULT_YARN_CONTAINER_SANDBOX)); - initializePolicyDir(); - super.initialize(conf); } @@ -223,6 +221,7 @@ public class JavaSandboxLinuxContainerRuntime OutputStream policyOutputStream = null; try { String containerID = ctx.getExecutionAttribute(CONTAINER_ID_STR); + initializePolicyDir(); Path policyFilePath = Files.createFile( Paths.get(policyFileDir.toString(), @@ -368,8 +367,12 @@ public class JavaSandboxLinuxContainerRuntime static final String STRIP_POLICY_FLAG = POLICY_APPEND_FLAG + "[^ ]+"; static final String CONTAINS_JAVA_CMD = "\\$" + JAVA_HOME + JAVA_CMD + ".*"; - static final String CHAINED_COMMAND_REGEX = - "^.*(&&.+$)|(\\|\\|.+$).*$"; //Matches any occurrences of '||' or '&&' + static final String MULTI_COMMAND_REGEX = + "(?s).*(" + //command read as single line + "(&[^>]|&&)|(\\|{1,2})|(\\|&)|" + //Matches '&','&&','|','||' and '|&' + "(`[^`]+`)|(\\$\\([^)]+\\))|" + //Matches occurrences of $() or `` + "(;)" + //Matches end of statement ';' + ").*"; static final String CLEAN_CMD_REGEX = "(" + SECURITY_FLAG + ")|" + "(" + STRIP_POLICY_FLAG + ")"; @@ -459,7 +462,7 @@ public class JavaSandboxLinuxContainerRuntime String command = commands.get(i); if(validateJavaHome(env.get(JAVA_HOME.name())) && command.matches(CONTAINS_JAVA_CMD) - && !command.matches(CHAINED_COMMAND_REGEX)){ + && !command.matches(MULTI_COMMAND_REGEX)){ command = command.replaceAll(CLEAN_CMD_REGEX, ""); String securityString = JVM_SECURITY_CMD + policyPath + " "; if(LOG.isDebugEnabled()) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b05bddb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestJavaSandboxLinuxContainerRuntime.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestJavaSandboxLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestJavaSandboxLinuxContainerRuntime.java index e482c8d..e10d0dd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestJavaSandboxLinuxContainerRuntime.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestJavaSandboxLinuxContainerRuntime.java @@ -47,7 +47,8 @@ import java.util.List; import java.util.Map; import static org.apache.hadoop.yarn.api.ApplicationConstants.Environment.JAVA_HOME; -import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.JavaSandboxLinuxContainerRuntime.NMContainerPolicyUtils.CHAINED_COMMAND_REGEX; +import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.JavaSandboxLinuxContainerRuntime.NMContainerPolicyUtils.LOG; +import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.JavaSandboxLinuxContainerRuntime.NMContainerPolicyUtils.MULTI_COMMAND_REGEX; import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.JavaSandboxLinuxContainerRuntime.NMContainerPolicyUtils.CLEAN_CMD_REGEX; import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.JavaSandboxLinuxContainerRuntime.NMContainerPolicyUtils.CONTAINS_JAVA_CMD; import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.JavaSandboxLinuxContainerRuntime.NMContainerPolicyUtils.POLICY_FILE; @@ -293,8 +294,21 @@ public class TestJavaSandboxLinuxContainerRuntime { @Test public void testChainedCmdRegex(){ - Assert.assertTrue("cmd1 && cmd2 || cmd3".matches(CHAINED_COMMAND_REGEX)); - Assert.assertFalse("cmd1 &> logfile".matches(CHAINED_COMMAND_REGEX)); + String[] multiCmds = { + "cmd1 && cmd2", + "cmd1 || cmd2", + "cmd1 `cmd2`", + "cmd1 $(cmd2)", + "cmd1; \\\n cmd2", + "cmd1; cmd2", + "cmd1|&cmd2", + "cmd1|cmd2", + "cmd1&cmd2" + }; + + Arrays.stream(multiCmds) + .forEach(cmd -> Assert.assertTrue(cmd.matches(MULTI_COMMAND_REGEX))); + Assert.assertFalse("cmd1 &> logfile".matches(MULTI_COMMAND_REGEX)); } @Test --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
