Repository: hadoop Updated Branches: refs/heads/branch-2.9 da6ea5c14 -> cfef479b7
YARN-7286. Add support for docker to have no capabilities. Contributed by Eric Badger (cherry picked from commit b7dee1f0608006e776624a9e4de39811d8aebc97) (cherry picked from commit 6ad6882343f73c285a00753dfef81ab68c7333ef) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/cfef479b Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/cfef479b Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/cfef479b Branch: refs/heads/branch-2.9 Commit: cfef479b73bc49d2c5906e990baf7f9afd53d39a Parents: da6ea5c Author: Jason Lowe <[email protected]> Authored: Thu Nov 2 09:50:55 2017 -0500 Committer: Jason Lowe <[email protected]> Committed: Thu Nov 2 11:02:37 2017 -0500 ---------------------------------------------------------------------- .../src/main/resources/yarn-default.xml | 3 +- .../runtime/DockerLinuxContainerRuntime.java | 30 ++++++++++++-- .../runtime/TestDockerContainerRuntime.java | 43 ++++++++++++++++++++ 3 files changed, 71 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/cfef479b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 937b7b0..46fb7c7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -1618,7 +1618,8 @@ <description>This configuration setting determines the capabilities assigned to docker containers when they are launched. While these may not be case-sensitive from a docker perspective, it is best to keep these - uppercase.</description> + uppercase. To run without any capabilites, set this value to + "none" or "NONE"</description> <name>yarn.nodemanager.runtime.linux.docker.capabilities</name> <value>CHOWN,DAC_OVERRIDE,FSETID,FOWNER,MKNOD,NET_RAW,SETGID,SETUID,SETFCAP,SETPCAP,NET_BIND_SERVICE,SYS_CHROOT,KILL,AUDIT_WRITE</value> </property> http://git-wip-us.apache.org/repos/asf/hadoop/blob/cfef479b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java index 2013306..5e3e15c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java @@ -54,6 +54,7 @@ import java.nio.file.Files; import java.nio.file.Paths; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -182,6 +183,7 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime { private boolean enableUserReMapping; private int userRemappingUidThreshold; private int userRemappingGidThreshold; + private Set<String> capabilities; /** * Return whether the given environment variables indicate that the operation @@ -279,6 +281,30 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime { userRemappingGidThreshold = conf.getInt( YarnConfiguration.NM_DOCKER_USER_REMAPPING_GID_THRESHOLD, YarnConfiguration.DEFAULT_NM_DOCKER_USER_REMAPPING_GID_THRESHOLD); + + capabilities = getDockerCapabilitiesFromConf(); + } + + private Set<String> getDockerCapabilitiesFromConf() throws + ContainerExecutionException { + Set<String> caps = new HashSet<>(Arrays.asList( + conf.getTrimmedStrings( + YarnConfiguration.NM_DOCKER_CONTAINER_CAPABILITIES, + YarnConfiguration.DEFAULT_NM_DOCKER_CONTAINER_CAPABILITIES))); + if(caps.contains("none") || caps.contains("NONE")) { + if(caps.size() > 1) { + String msg = "Mixing capabilities with the none keyword is" + + " not supported"; + throw new ContainerExecutionException(msg); + } + caps = Collections.emptySet(); + } + + return caps; + } + + public Set<String> getCapabilities() { + return capabilities; } @Override @@ -551,10 +577,6 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime { LOCALIZED_RESOURCES); @SuppressWarnings("unchecked") List<String> userLocalDirs = ctx.getExecutionAttribute(USER_LOCAL_DIRS); - Set<String> capabilities = new HashSet<>(Arrays.asList( - conf.getTrimmedStrings( - YarnConfiguration.NM_DOCKER_CONTAINER_CAPABILITIES, - YarnConfiguration.DEFAULT_NM_DOCKER_CONTAINER_CAPABILITIES))); @SuppressWarnings("unchecked") DockerRunCommand runCommand = new DockerRunCommand(containerIdStr, http://git-wip-us.apache.org/repos/asf/hadoop/blob/cfef479b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java index 8c3b947..aef94a7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java @@ -58,6 +58,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Set; @@ -1150,4 +1151,46 @@ public class TestDockerContainerRuntime { } } } + + @Test + public void testDockerCapabilities() + throws ContainerExecutionException, PrivilegedOperationException, + IOException { + DockerLinuxContainerRuntime runtime = new DockerLinuxContainerRuntime( + mockExecutor, mockCGroupsHandler); + try { + conf.setStrings(YarnConfiguration.NM_DOCKER_CONTAINER_CAPABILITIES, + "none", "CHOWN", "DAC_OVERRIDE"); + runtime.initialize(conf); + Assert.fail("Initialize didn't fail with invalid capabilities " + + "'none', 'CHOWN', 'DAC_OVERRIDE'"); + } catch (ContainerExecutionException e) { + } + + try { + conf.setStrings(YarnConfiguration.NM_DOCKER_CONTAINER_CAPABILITIES, + "CHOWN", "DAC_OVERRIDE", "NONE"); + runtime.initialize(conf); + Assert.fail("Initialize didn't fail with invalid capabilities " + + "'CHOWN', 'DAC_OVERRIDE', 'NONE'"); + } catch (ContainerExecutionException e) { + } + + conf.setStrings(YarnConfiguration.NM_DOCKER_CONTAINER_CAPABILITIES, + "NONE"); + runtime.initialize(conf); + Assert.assertEquals(0, runtime.getCapabilities().size()); + + conf.setStrings(YarnConfiguration.NM_DOCKER_CONTAINER_CAPABILITIES, + "none"); + runtime.initialize(conf); + Assert.assertEquals(0, runtime.getCapabilities().size()); + + conf.setStrings(YarnConfiguration.NM_DOCKER_CONTAINER_CAPABILITIES, + "CHOWN", "DAC_OVERRIDE"); + runtime.initialize(conf); + Iterator<String> it = runtime.getCapabilities().iterator(); + Assert.assertEquals("CHOWN", it.next()); + Assert.assertEquals("DAC_OVERRIDE", it.next()); + } } --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
