Repository: hadoop
Updated Branches:
refs/heads/branch-3.0 2ba71afcd -> b7dee1f06
YARN-7286. Add support for docker to have no capabilities. Contributed by Eric
Badger
(cherry picked from commit d00b6f7c1ff2d7569ae9efdc6823ebcfb86ef2d4)
Conflicts:
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b7dee1f0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b7dee1f0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b7dee1f0
Branch: refs/heads/branch-3.0
Commit: b7dee1f0608006e776624a9e4de39811d8aebc97
Parents: 2ba71afc
Author: Jason Lowe <[email protected]>
Authored: Thu Nov 2 09:37:17 2017 -0500
Committer: Jason Lowe <[email protected]>
Committed: Thu Nov 2 09:47:26 2017 -0500
----------------------------------------------------------------------
.../src/main/resources/yarn-default.xml | 3 +-
.../runtime/DockerLinuxContainerRuntime.java | 30 ++++++++++++--
.../runtime/TestDockerContainerRuntime.java | 43 ++++++++++++++++++++
3 files changed, 71 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b7dee1f0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
----------------------------------------------------------------------
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 09c782e..9abf600 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -1607,7 +1607,8 @@
<description>This configuration setting determines the capabilities
assigned to docker containers when they are launched. While these may not
be case-sensitive from a docker perspective, it is best to keep these
- uppercase.</description>
+ uppercase. To run without any capabilites, set this value to
+ "none" or "NONE"</description>
<name>yarn.nodemanager.runtime.linux.docker.capabilities</name>
<value>CHOWN,DAC_OVERRIDE,FSETID,FOWNER,MKNOD,NET_RAW,SETGID,SETUID,SETFCAP,SETPCAP,NET_BIND_SERVICE,SYS_CHROOT,KILL,AUDIT_WRITE</value>
</property>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b7dee1f0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
----------------------------------------------------------------------
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
index 2013306..5e3e15c 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
@@ -54,6 +54,7 @@ import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
@@ -182,6 +183,7 @@ public class DockerLinuxContainerRuntime implements
LinuxContainerRuntime {
private boolean enableUserReMapping;
private int userRemappingUidThreshold;
private int userRemappingGidThreshold;
+ private Set<String> capabilities;
/**
* Return whether the given environment variables indicate that the operation
@@ -279,6 +281,30 @@ public class DockerLinuxContainerRuntime implements
LinuxContainerRuntime {
userRemappingGidThreshold = conf.getInt(
YarnConfiguration.NM_DOCKER_USER_REMAPPING_GID_THRESHOLD,
YarnConfiguration.DEFAULT_NM_DOCKER_USER_REMAPPING_GID_THRESHOLD);
+
+ capabilities = getDockerCapabilitiesFromConf();
+ }
+
+ private Set<String> getDockerCapabilitiesFromConf() throws
+ ContainerExecutionException {
+ Set<String> caps = new HashSet<>(Arrays.asList(
+ conf.getTrimmedStrings(
+ YarnConfiguration.NM_DOCKER_CONTAINER_CAPABILITIES,
+ YarnConfiguration.DEFAULT_NM_DOCKER_CONTAINER_CAPABILITIES)));
+ if(caps.contains("none") || caps.contains("NONE")) {
+ if(caps.size() > 1) {
+ String msg = "Mixing capabilities with the none keyword is" +
+ " not supported";
+ throw new ContainerExecutionException(msg);
+ }
+ caps = Collections.emptySet();
+ }
+
+ return caps;
+ }
+
+ public Set<String> getCapabilities() {
+ return capabilities;
}
@Override
@@ -551,10 +577,6 @@ public class DockerLinuxContainerRuntime implements
LinuxContainerRuntime {
LOCALIZED_RESOURCES);
@SuppressWarnings("unchecked")
List<String> userLocalDirs = ctx.getExecutionAttribute(USER_LOCAL_DIRS);
- Set<String> capabilities = new HashSet<>(Arrays.asList(
- conf.getTrimmedStrings(
- YarnConfiguration.NM_DOCKER_CONTAINER_CAPABILITIES,
- YarnConfiguration.DEFAULT_NM_DOCKER_CONTAINER_CAPABILITIES)));
@SuppressWarnings("unchecked")
DockerRunCommand runCommand = new DockerRunCommand(containerIdStr,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b7dee1f0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
----------------------------------------------------------------------
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
index fbfee54..be26aa5 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
@@ -57,6 +57,7 @@ import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -1148,4 +1149,46 @@ public class TestDockerContainerRuntime {
}
}
}
+
+ @Test
+ public void testDockerCapabilities()
+ throws ContainerExecutionException, PrivilegedOperationException,
+ IOException {
+ DockerLinuxContainerRuntime runtime = new DockerLinuxContainerRuntime(
+ mockExecutor, mockCGroupsHandler);
+ try {
+ conf.setStrings(YarnConfiguration.NM_DOCKER_CONTAINER_CAPABILITIES,
+ "none", "CHOWN", "DAC_OVERRIDE");
+ runtime.initialize(conf);
+ Assert.fail("Initialize didn't fail with invalid capabilities " +
+ "'none', 'CHOWN', 'DAC_OVERRIDE'");
+ } catch (ContainerExecutionException e) {
+ }
+
+ try {
+ conf.setStrings(YarnConfiguration.NM_DOCKER_CONTAINER_CAPABILITIES,
+ "CHOWN", "DAC_OVERRIDE", "NONE");
+ runtime.initialize(conf);
+ Assert.fail("Initialize didn't fail with invalid capabilities " +
+ "'CHOWN', 'DAC_OVERRIDE', 'NONE'");
+ } catch (ContainerExecutionException e) {
+ }
+
+ conf.setStrings(YarnConfiguration.NM_DOCKER_CONTAINER_CAPABILITIES,
+ "NONE");
+ runtime.initialize(conf);
+ Assert.assertEquals(0, runtime.getCapabilities().size());
+
+ conf.setStrings(YarnConfiguration.NM_DOCKER_CONTAINER_CAPABILITIES,
+ "none");
+ runtime.initialize(conf);
+ Assert.assertEquals(0, runtime.getCapabilities().size());
+
+ conf.setStrings(YarnConfiguration.NM_DOCKER_CONTAINER_CAPABILITIES,
+ "CHOWN", "DAC_OVERRIDE");
+ runtime.initialize(conf);
+ Iterator<String> it = runtime.getCapabilities().iterator();
+ Assert.assertEquals("CHOWN", it.next());
+ Assert.assertEquals("DAC_OVERRIDE", it.next());
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
