Repository: hadoop Updated Branches: refs/heads/trunk c114da5e6 -> d3b1c6319
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerCommandExecutor.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerCommandExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerCommandExecutor.java index 60fce40..05b44b8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerCommandExecutor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerCommandExecutor.java @@ -114,8 +114,10 @@ public class TestDockerCommandExecutor { assertEquals(1, ops.size()); assertEquals(PrivilegedOperation.OperationType.RUN_DOCKER_CMD.name(), ops.get(0).getOperationType().name()); - assertEquals(1, dockerCommands.size()); - assertEquals("rm " + MOCK_CONTAINER_ID, dockerCommands.get(0)); + assertEquals(3, dockerCommands.size()); + assertEquals("[docker-command-execution]", dockerCommands.get(0)); + assertEquals(" docker-command=rm", dockerCommands.get(1)); + assertEquals(" name=" + MOCK_CONTAINER_ID, dockerCommands.get(2)); } @Test @@ -130,8 +132,10 @@ public class TestDockerCommandExecutor { assertEquals(1, ops.size()); assertEquals(PrivilegedOperation.OperationType.RUN_DOCKER_CMD.name(), ops.get(0).getOperationType().name()); - assertEquals(1, dockerCommands.size()); - assertEquals("stop " + MOCK_CONTAINER_ID, dockerCommands.get(0)); + assertEquals(3, dockerCommands.size()); + assertEquals("[docker-command-execution]", dockerCommands.get(0)); + assertEquals(" docker-command=stop", dockerCommands.get(1)); + assertEquals(" name=" + MOCK_CONTAINER_ID, dockerCommands.get(2)); } @Test @@ -147,9 +151,12 @@ public class TestDockerCommandExecutor { assertEquals(1, ops.size()); assertEquals(PrivilegedOperation.OperationType.RUN_DOCKER_CMD.name(), ops.get(0).getOperationType().name()); - assertEquals(1, dockerCommands.size()); - assertEquals("inspect --format='{{.State.Status}}' " + MOCK_CONTAINER_ID, - dockerCommands.get(0)); + assertEquals(4, dockerCommands.size()); + assertEquals("[docker-command-execution]", dockerCommands.get(0)); + assertEquals(" docker-command=inspect", dockerCommands.get(1)); + assertEquals(" format={{.State.Status}}", dockerCommands.get(2)); + assertEquals(" name=" + MOCK_CONTAINER_ID, dockerCommands.get(3)); + } @Test @@ -165,8 +172,10 @@ public class TestDockerCommandExecutor { assertEquals(1, ops.size()); assertEquals(PrivilegedOperation.OperationType.RUN_DOCKER_CMD.name(), ops.get(0).getOperationType().name()); - assertEquals(1, dockerCommands.size()); - assertEquals("pull " + MOCK_IMAGE_NAME, dockerCommands.get(0)); + assertEquals(3, dockerCommands.size()); + assertEquals("[docker-command-execution]", dockerCommands.get(0)); + assertEquals(" docker-command=pull", dockerCommands.get(1)); + assertEquals(" image=" + MOCK_IMAGE_NAME, dockerCommands.get(2)); } @Test @@ -182,8 +191,12 @@ public class TestDockerCommandExecutor { assertEquals(1, ops.size()); assertEquals(PrivilegedOperation.OperationType.RUN_DOCKER_CMD.name(), ops.get(0).getOperationType().name()); - assertEquals(1, dockerCommands.size()); - assertEquals("load --i=" + MOCK_LOCAL_IMAGE_NAME, dockerCommands.get(0)); + assertEquals(3, dockerCommands.size()); + assertEquals("[docker-command-execution]", dockerCommands.get(0)); + assertEquals(" docker-command=load", dockerCommands.get(1)); + assertEquals(" image=" + MOCK_LOCAL_IMAGE_NAME, dockerCommands.get(2)); + + } @Test http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerInspectCommand.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerInspectCommand.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerInspectCommand.java index 619f202..4092e6c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerInspectCommand.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerInspectCommand.java @@ -18,6 +18,8 @@ package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker; import static org.junit.Assert.assertEquals; + +import org.apache.hadoop.util.StringUtils; import org.junit.Before; import org.junit.Test; @@ -44,16 +46,29 @@ public class TestDockerInspectCommand { @Test public void testGetContainerStatus() throws Exception { dockerInspectCommand.getContainerStatus(); - assertEquals("inspect --format='{{.State.Status}}' foo", - dockerInspectCommand.getCommandWithArguments()); + assertEquals("inspect", StringUtils.join(",", + dockerInspectCommand.getDockerCommandWithArguments() + .get("docker-command"))); + assertEquals("{{.State.Status}}", StringUtils.join(",", + dockerInspectCommand.getDockerCommandWithArguments().get("format"))); + assertEquals("foo", StringUtils.join(",", + dockerInspectCommand.getDockerCommandWithArguments().get("name"))); + assertEquals(3, + dockerInspectCommand.getDockerCommandWithArguments().size()); } @Test public void testGetIpAndHost() throws Exception { dockerInspectCommand.getIpAndHost(); - assertEquals( - "inspect --format='{{range(.NetworkSettings.Networks)}}{{.IPAddress}}" - + ",{{end}}{{.Config.Hostname}}' foo", - dockerInspectCommand.getCommandWithArguments()); + assertEquals("inspect", StringUtils.join(",", + dockerInspectCommand.getDockerCommandWithArguments() + .get("docker-command"))); + assertEquals("{{range(.NetworkSettings.Networks)}}" + + "{{.IPAddress}},{{end}}{{.Config.Hostname}}", StringUtils.join(",", + dockerInspectCommand.getDockerCommandWithArguments().get("format"))); + assertEquals("foo", StringUtils.join(",", + dockerInspectCommand.getDockerCommandWithArguments().get("name"))); + assertEquals(3, + dockerInspectCommand.getDockerCommandWithArguments().size()); } -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerLoadCommand.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerLoadCommand.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerLoadCommand.java index 85fa0f8..e5bff26 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerLoadCommand.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerLoadCommand.java @@ -16,6 +16,7 @@ */ package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker; +import org.apache.hadoop.util.StringUtils; import org.junit.Before; import org.junit.Test; @@ -42,7 +43,11 @@ public class TestDockerLoadCommand { @Test public void testGetCommandWithArguments() { - assertEquals("load --i=foo", - dockerLoadCommand.getCommandWithArguments()); + assertEquals("load", StringUtils.join(",", + dockerLoadCommand.getDockerCommandWithArguments() + .get("docker-command"))); + assertEquals("foo", StringUtils.join(",", + dockerLoadCommand.getDockerCommandWithArguments().get("image"))); + assertEquals(2, dockerLoadCommand.getDockerCommandWithArguments().size()); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerPullCommand.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerPullCommand.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerPullCommand.java index 89157ff..ccf7000 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerPullCommand.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerPullCommand.java @@ -16,6 +16,7 @@ */ package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker; +import org.apache.hadoop.util.StringUtils; import org.junit.Before; import org.junit.Test; @@ -42,7 +43,12 @@ public class TestDockerPullCommand { @Test public void testGetCommandWithArguments() { - assertEquals("pull foo", dockerPullCommand.getCommandWithArguments()); + assertEquals("pull", StringUtils.join(",", + dockerPullCommand.getDockerCommandWithArguments() + .get("docker-command"))); + assertEquals("foo", StringUtils.join(",", + dockerPullCommand.getDockerCommandWithArguments().get("image"))); + assertEquals(2, dockerPullCommand.getDockerCommandWithArguments().size()); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerRmCommand.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerRmCommand.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerRmCommand.java index d1b9904..a8d4bdd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerRmCommand.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerRmCommand.java @@ -17,6 +17,8 @@ package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker; import static org.junit.Assert.assertEquals; + +import org.apache.hadoop.util.StringUtils; import org.junit.Before; import org.junit.Test; @@ -42,7 +44,11 @@ public class TestDockerRmCommand { @Test public void testGetCommandWithArguments() { - assertEquals("rm foo", dockerRmCommand.getCommandWithArguments()); + assertEquals("rm", StringUtils.join(",", + dockerRmCommand.getDockerCommandWithArguments().get("docker-command"))); + assertEquals("foo", StringUtils.join(",", + dockerRmCommand.getDockerCommandWithArguments().get("name"))); + assertEquals(2, dockerRmCommand.getDockerCommandWithArguments().size()); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerRunCommand.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerRunCommand.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerRunCommand.java index 85bccd2..e51d7ec 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerRunCommand.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerRunCommand.java @@ -16,6 +16,7 @@ */ package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker; +import org.apache.hadoop.util.StringUtils; import org.junit.Before; import org.junit.Test; @@ -56,8 +57,24 @@ public class TestDockerRunCommand { commands.add("launch_command"); dockerRunCommand.setOverrideCommandWithArgs(commands); dockerRunCommand.removeContainerOnExit(); - assertEquals("run --name=foo --user=user_id --device=source:dest --rm " - + "image_name launch_command", - dockerRunCommand.getCommandWithArguments()); + + assertEquals("run", StringUtils.join(",", + dockerRunCommand.getDockerCommandWithArguments() + .get("docker-command"))); + assertEquals("foo", StringUtils.join(",", + dockerRunCommand.getDockerCommandWithArguments().get("name"))); + assertEquals("user_id", StringUtils.join(",", + dockerRunCommand.getDockerCommandWithArguments().get("user"))); + assertEquals("image_name", StringUtils.join(",", + dockerRunCommand.getDockerCommandWithArguments().get("image"))); + + assertEquals("source:dest", StringUtils.join(",", + dockerRunCommand.getDockerCommandWithArguments().get("devices"))); + assertEquals("true", StringUtils + .join(",", dockerRunCommand.getDockerCommandWithArguments().get("rm"))); + assertEquals("launch_command", StringUtils.join(",", + dockerRunCommand.getDockerCommandWithArguments() + .get("launch-command"))); + assertEquals(7, dockerRunCommand.getDockerCommandWithArguments().size()); } -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerStopCommand.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerStopCommand.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerStopCommand.java index c9743f3..efbde77 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerStopCommand.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerStopCommand.java @@ -21,6 +21,8 @@ package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker; import static org.junit.Assert.assertEquals; + +import org.apache.hadoop.util.StringUtils; import org.junit.Before; import org.junit.Test; @@ -48,8 +50,13 @@ public class TestDockerStopCommand { @Test public void testSetGracePeriod() throws Exception { dockerStopCommand.setGracePeriod(GRACE_PERIOD); - assertEquals("stop foo --time=10", - dockerStopCommand.getCommandWithArguments()); - + assertEquals("stop", StringUtils.join(",", + dockerStopCommand.getDockerCommandWithArguments() + .get("docker-command"))); + assertEquals("foo", StringUtils.join(",", + dockerStopCommand.getDockerCommandWithArguments().get("name"))); + assertEquals("10", StringUtils.join(",", + dockerStopCommand.getDockerCommandWithArguments().get("time"))); + assertEquals(3, dockerStopCommand.getDockerCommandWithArguments().size()); } -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md index 23f4134..36c391a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md @@ -167,7 +167,24 @@ The following properties are required to enable Docker support: |Configuration Name | Description | |:---- |:---- | | `yarn.nodemanager.linux-container-executor.group` | The Unix group of the NodeManager. It should match the yarn.nodemanager.linux-container-executor.group in the yarn-site.xml file. | -| `feature.docker.enabled` | Must be 0 or 1. 0 means launching Docker containers is disabled. 1 means launching Docker containers is allowed. | + +The container-executor.cfg must contain a section to determine the capabilities that containers +are allowed. It contains the following properties: + +|Configuration Name | Description | +|:---- |:---- | +| `module.enabled` | Must be "true" or "false" to enable or disable launching Docker containers respectively. Default value is 0. | +| `docker.binary` | The binary used to launch Docker containers. /usr/bin/docker by default. | +| `docker.allowed.capabilities` | Comma separated capabilities that containers are allowed to add. By default no capabilities are allowed to be added. | +| `docker.allowed.devices` | Comma separated devices that containers are allowed to mount. By default no devices are allowed to be added. | +| `docker.allowed.networks` | Comma separated networks that containers are allowed to use. If no network is specified when launching the container, the default Docker network will be used. | +| `docker.allowed.ro-mounts` | Comma separated directories that containers are allowed to mount in read-only mode. By default, no directories are allowed to mounted. | +| `docker.allowed.rw-mounts` | Comma separated directories that containers are allowed to mount in read-write mode. By default, no directories are allowed to mounted. | +| `docker.privileged-containers.enabled` | Set to 1 or 0 to enable or disable launching privileged containers. Default value is 0. | + +Please note that if you wish to run Docker containers that require access to the YARN local directories, you must add them to the docker.allowed.rw-mounts list. + +In addition, containers are not permitted to mount any parent of the container-executor.cfg directory in read-write mode. The following properties are optional: @@ -176,9 +193,21 @@ The following properties are optional: | `min.user.id` | The minimum UID that is allowed to launch applications. The default is no minimum | | `banned.users` | A comma-separated list of usernames who should not be allowed to launch applications. The default setting is: yarn, mapred, hdfs, and bin. | | `allowed.system.users` | A comma-separated list of usernames who should be allowed to launch applications even if their UIDs are below the configured minimum. If a user appears in allowed.system.users and banned.users, the user will be considered banned. | -| `docker.binary` | The path to the Docker binary. The default is "docker". | | `feature.tc.enabled` | Must be 0 or 1. 0 means traffic control commands are disabled. 1 means traffic control commands are allowed. | +Part of a container-executor.cfg which allows Docker containers to be launched is below: + +``` +yarn.nodemanager.linux-container-executor.group=yarn +[docker] + module.enabled=true + docker.allowed.capabilities=SYS_CHROOT,MKNOD,SETFCAP,SETPCAP,FSETID,CHOWN,AUDIT_WRITE,SETGID,NET_RAW,FOWNER,SETUID,DAC_OVERRIDE,KILL,NET_BIND_SERVICE + docker.allowed.networks=bridge,host,none + docker.allowed.ro-mounts=/sys/fs/cgroup + docker.allowed.rw-mounts=/var/hadoop/yarn/local-dir,/var/hadoop/yarn/log-dir + +``` + Docker Image Requirements ------------------------- --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
