Repository: hadoop
Updated Branches:
refs/heads/trunk 0d8406135 -> 33e0df4b3
YARN-8986. Added port publish for Docker container running with bridge network.
Contributed by Charo Zhang
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/33e0df4b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/33e0df4b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/33e0df4b
Branch: refs/heads/trunk
Commit: 33e0df4b3558bea4a7977695d2c564af9a0c57e7
Parents: 0d84061
Author: Eric Yang <[email protected]>
Authored: Mon Nov 26 19:45:05 2018 -0500
Committer: Eric Yang <[email protected]>
Committed: Mon Nov 26 19:45:05 2018 -0500
----------------------------------------------------------------------
.../runtime/DockerLinuxContainerRuntime.java | 26 ++++++
.../linux/runtime/docker/DockerRunCommand.java | 6 ++
.../container-executor/impl/utils/docker-util.c | 72 +++++++++++++++
.../container-executor/impl/utils/docker-util.h | 1 +
.../test/utils/test_docker_util.cc | 94 ++++++++++++++++----
.../runtime/docker/TestDockerRunCommand.java | 9 +-
.../src/site/markdown/DockerContainers.md | 1 +
7 files changed, 189 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/33e0df4b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
----------------------------------------------------------------------
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
index 83460fc..15ff0ff 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
@@ -123,6 +123,13 @@ import static
org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.r
* property.
* </li>
* <li>
+ * {@code YARN_CONTAINER_RUNTIME_DOCKER_PORTS_MAPPING} allows users to
+ * specify ports mapping for the bridge network Docker container. The value
+ * of the environment variable should be a comma-separated list of ports
+ * mapping. It's the same to "-p" option for the Docker run command. If the
+ * value is empty, "-P" will be added.
+ * </li>
+ * <li>
* {@code YARN_CONTAINER_RUNTIME_DOCKER_CONTAINER_PID_NAMESPACE}
* controls which PID namespace will be used by the Docker container. By
* default, each Docker container has its own PID namespace. To share the
@@ -210,6 +217,10 @@ public class DockerLinuxContainerRuntime implements
LinuxContainerRuntime {
"(:(r[ow]|(r[ow][+])?(r?shared|r?slave|r?private)))?(?:,|$)");
private static final Pattern TMPFS_MOUNT_PATTERN = Pattern.compile(
"^/[^:\\x00]+$");
+ public static final String PORTS_MAPPING_PATTERN =
+ "^:[0-9]+|^[0-9]+:[0-9]+|^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]" +
+ "|25[0-5])\\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])" +
+ ":[0-9]+:[0-9]+$";
private static final int HOST_NAME_LENGTH = 64;
private static final String DEFAULT_PROCFS = "/proc";
@@ -241,6 +252,9 @@ public class DockerLinuxContainerRuntime implements
LinuxContainerRuntime {
public static final String ENV_DOCKER_CONTAINER_DELAYED_REMOVAL =
"YARN_CONTAINER_RUNTIME_DOCKER_DELAYED_REMOVAL";
@InterfaceAudience.Private
+ public static final String ENV_DOCKER_CONTAINER_PORTS_MAPPING =
+ "YARN_CONTAINER_RUNTIME_DOCKER_PORTS_MAPPING";
+ @InterfaceAudience.Private
public static final String ENV_DOCKER_CONTAINER_YARN_SYSFS =
"YARN_CONTAINER_RUNTIME_YARN_SYSFS_ENABLE";
public static final String YARN_SYSFS_PATH =
@@ -873,6 +887,18 @@ public class DockerLinuxContainerRuntime implements
LinuxContainerRuntime {
setHostname(runCommand, containerIdStr, network, hostname);
+ // Add ports mapping value.
+ if (environment.containsKey(ENV_DOCKER_CONTAINER_PORTS_MAPPING)) {
+ String portsMapping =
environment.get(ENV_DOCKER_CONTAINER_PORTS_MAPPING);
+ for (String mapping:portsMapping.split(",")) {
+ if (!Pattern.matches(PORTS_MAPPING_PATTERN, mapping)) {
+ throw new ContainerExecutionException(
+ "Invalid port mappings: " + mapping);
+ }
+ runCommand.addPortsMapping(mapping);
+ }
+ }
+
runCommand.setCapabilities(capabilities);
runCommand.addAllReadWriteMountLocations(containerLogDirs);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/33e0df4b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java
----------------------------------------------------------------------
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java
index aac8224..f4f3a9c 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java
@@ -159,6 +159,12 @@ public class DockerRunCommand extends DockerCommand {
return this;
}
+ /* Ports mapping for bridge network, -p */
+ public DockerRunCommand addPortsMapping(String mapping) {
+ super.addCommandArguments("ports-mapping", mapping);
+ return this;
+ }
+
public DockerRunCommand groupAdd(String[] groups) {
super.addCommandArguments("group-add", String.join(",", groups));
return this;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/33e0df4b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c
----------------------------------------------------------------------
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c
index c0aa80b..e7a09a6 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c
@@ -167,6 +167,13 @@ static int is_volume_name(const char *volume_name) {
return execute_regex_match(regex_str, volume_name) == 0;
}
+static int is_valid_ports_mapping(const char *ports_mapping) {
+ const char *regex_str =
"^:[0-9]+|^[0-9]+:[0-9]+|^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.)"
+
"{3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]):[0-9]+:[0-9]+$";
+ // execute_regex_match return 0 is matched success
+ return execute_regex_match(regex_str, ports_mapping) == 0;
+}
+
static int is_volume_name_matched_by_regex(const char* requested, const char*
pattern) {
// execute_regex_match return 0 is matched success
return is_volume_name(requested) && (execute_regex_match(pattern +
sizeof("regex:"), requested) == 0);
@@ -314,6 +321,8 @@ const char *get_docker_error_message(const int error_code) {
return "Unknown docker command";
case INVALID_DOCKER_NETWORK:
return "Invalid docker network";
+ case INVALID_DOCKER_PORTS_MAPPING:
+ return "Invalid docker ports mapping";
case INVALID_DOCKER_CAPABILITY:
return "Invalid docker capability";
case PRIVILEGED_CONTAINERS_DISABLED:
@@ -936,6 +945,64 @@ static int set_network(const struct configuration
*command_config,
return ret;
}
+static int add_ports_mapping_to_command(const struct configuration
*command_config, args *args) {
+ int i = 0, ret = 0;
+ char *network_type = (char*) malloc(128);
+ char *docker_network_command = NULL;
+ char *docker_binary = get_docker_binary(command_config);
+ char *network_name = get_configuration_value("net",
DOCKER_COMMAND_FILE_SECTION, command_config);
+ char **ports_mapping_values =
get_configuration_values_delimiter("ports-mapping",
DOCKER_COMMAND_FILE_SECTION, command_config, ",");
+ if (network_name != NULL) {
+ docker_network_command = make_string("%s network inspect %s
--format='{{.Driver}}'", docker_binary, network_name);
+ FILE* docker_network = popen(docker_network_command, "r");
+ ret = fscanf(docker_network, "%s", network_type);
+ if (pclose (docker_network) != 0 || ret <= 0) {
+ fprintf (ERRORFILE, "Could not inspect docker network to get type
%s.\n", docker_network_command);
+ goto cleanup;
+ }
+ // other network type exit successfully without ports mapping
+ if (strcasecmp(network_type, "bridge") != 0) {
+ ret = 0;
+ goto cleanup;
+ }
+ // add -P when not configure ports mapping
+ if (ports_mapping_values == NULL) {
+ ret = add_to_args(args, "-P");
+ if (ret != 0) {
+ ret = BUFFER_TOO_SMALL;
+ }
+ }
+ }
+ // add -p when configure ports mapping
+ if (ports_mapping_values != NULL) {
+ for (i = 0; ports_mapping_values[i] != NULL; i++) {
+ if (!is_valid_ports_mapping(ports_mapping_values[i])) {
+ fprintf (ERRORFILE, "Invalid port mappings: %s.\n",
ports_mapping_values[i]);
+ ret = INVALID_DOCKER_PORTS_MAPPING;
+ break;
+ }
+ ret = add_to_args(args, "-p");
+ if (ret != 0) {
+ ret = BUFFER_TOO_SMALL;
+ break;
+ }
+ ret = add_to_args(args, ports_mapping_values[i]);
+ if (ret != 0) {
+ ret = BUFFER_TOO_SMALL;
+ break;
+ }
+ }
+ }
+
+cleanup:
+ free(network_type);
+ free(docker_binary);
+ free(network_name);
+ free(docker_network_command);
+ free_values(ports_mapping_values);
+ return ret;
+}
+
static int set_pid_namespace(const struct configuration *command_config,
const struct configuration *conf, args *args) {
char *value = get_configuration_value("pid", DOCKER_COMMAND_FILE_SECTION,
@@ -1550,6 +1617,11 @@ int get_docker_run_command(const char *command_file,
const struct configuration
goto free_and_exit;
}
+ ret = add_ports_mapping_to_command(&command_config, args);
+ if (ret != 0) {
+ goto free_and_exit;
+ }
+
ret = set_pid_namespace(&command_config, conf, args);
if (ret != 0) {
goto free_and_exit;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/33e0df4b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.h
----------------------------------------------------------------------
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.h
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.h
index fc7b4cc..3cff565 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.h
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.h
@@ -53,6 +53,7 @@ enum docker_error_codes {
INVALID_DOCKER_INSPECT_FORMAT,
UNKNOWN_DOCKER_COMMAND,
INVALID_DOCKER_NETWORK,
+ INVALID_DOCKER_PORTS_MAPPING,
INVALID_DOCKER_CAPABILITY,
PRIVILEGED_CONTAINERS_DISABLED,
INVALID_DOCKER_MOUNT,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/33e0df4b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test_docker_util.cc
----------------------------------------------------------------------
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test_docker_util.cc
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test_docker_util.cc
index 2817e0c..6baab0f 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test_docker_util.cc
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test_docker_util.cc
@@ -502,6 +502,65 @@ namespace ContainerExecutor {
free_configuration(&container_cfg);
}
+ TEST_F(TestDockerUtil, test_add_ports_mapping_to_command) {
+ struct args buff = ARGS_INITIAL_VALUE;
+ int ret = 0;
+ std::vector<std::pair<std::string, std::string> > file_cmd_vec;
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run\n
ports-mapping=127.0.0.1:8080:80,1234:1234,:2222",
+ "-p 127.0.0.1:8080:80 -p 1234:1234 -p :2222"));
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run\n
ports-mapping=1234:1234,:2222",
+ "-p 1234:1234 -p :2222"));
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run\n
ports-mapping=:2222", "-p :2222"));
+
+ std::vector<std::pair<std::string, std::string> >::const_iterator itr;
+ for (itr = file_cmd_vec.begin(); itr != file_cmd_vec.end(); ++itr) {
+ struct configuration cmd_cfg;
+ write_command_file(itr->first);
+ ret = read_config(docker_command_file.c_str(), &cmd_cfg);
+ if (ret != 0) {
+ FAIL();
+ }
+ ret = add_ports_mapping_to_command(&cmd_cfg, &buff);
+ char *actual = flatten(&buff);
+ ASSERT_EQ(0, ret) << "error message: " << get_docker_error_message(ret)
<< " for input " << itr->first;
+ ASSERT_STREQ(itr->second.c_str(), actual);
+ reset_args(&buff);
+ free(actual);
+ free_configuration(&cmd_cfg);
+ }
+ struct configuration cmd_cfg_1;
+ write_command_file("[docker-command-execution]\n docker-command=run\n
ports-mapping=327.0.0.1:8080:80,1234:1234,:2222");
+ ret = read_config(docker_command_file.c_str(), &cmd_cfg_1);
+ if (ret != 0) {
+ FAIL();
+ }
+ ret = add_ports_mapping_to_command(&cmd_cfg_1, &buff);
+ ASSERT_EQ(INVALID_DOCKER_PORTS_MAPPING, ret);
+ reset_args(&buff);
+
+ write_command_file("[docker-command-execution]\n docker-command=run\n
ports-mapping=127.0.0.1:8080:80,12s4:1234,:2222");
+ ret = read_config(docker_command_file.c_str(), &cmd_cfg_1);
+ if (ret != 0) {
+ FAIL();
+ }
+ ret = add_ports_mapping_to_command(&cmd_cfg_1, &buff);
+ ASSERT_EQ(INVALID_DOCKER_PORTS_MAPPING, ret);
+ reset_args(&buff);
+
+ write_command_file("[docker-command-execution]\n docker-command=run\n
ports-mapping=127.0.0.1:8080:80,1234:1234,:s2s2");
+ ret = read_config(docker_command_file.c_str(), &cmd_cfg_1);
+ if (ret != 0) {
+ FAIL();
+ }
+ ret = add_ports_mapping_to_command(&cmd_cfg_1, &buff);
+ ASSERT_EQ(INVALID_DOCKER_PORTS_MAPPING, ret);
+ reset_args(&buff);
+ free_configuration(&cmd_cfg_1);
+ }
+
TEST_F(TestDockerUtil, test_set_pid_namespace) {
struct configuration container_cfg, cmd_cfg;
struct args buff = ARGS_INITIAL_VALUE;
@@ -1305,10 +1364,10 @@ namespace ContainerExecutor {
"[docker-command-execution]\n"
" docker-command=run\n name=container_e1_12312_11111_02_000001\n
image=hadoop/docker-image\n user=nobody\n hostname=host-id\n"
"
mounts=/var/log:/var/log:ro,/var/lib:/lib:ro,/usr/bin/cut:/usr/bin/cut:ro,/tmp:/tmp:rw\n"
- " network=bridge\n devices=/dev/test:/dev/test\n net=bridge\n"
+ " network=bridge\n devices=/dev/test:/dev/test\n"
" cap-add=CHOWN,SETUID\n cgroup-parent=ctr-cgroup\n
detach=true\n rm=true\n"
" launch-command=bash,test_script.sh,arg1,arg2",
- "run --name=container_e1_12312_11111_02_000001 --user=nobody -d --rm
--net=bridge -v /var/log:/var/log:ro -v /var/lib:/lib:ro"
+ "run --name=container_e1_12312_11111_02_000001 --user=nobody -d --rm
-v /var/log:/var/log:ro -v /var/lib:/lib:ro"
" -v /usr/bin/cut:/usr/bin/cut:ro -v /tmp:/tmp:rw
--cgroup-parent=ctr-cgroup --cap-drop=ALL --cap-add=CHOWN "
"--cap-add=SETUID --hostname=host-id --device=/dev/test:/dev/test
hadoop/docker-image bash"
" test_script.sh arg1 arg2"));
@@ -1316,10 +1375,9 @@ namespace ContainerExecutor {
"[docker-command-execution]\n"
" docker-command=run\n name=container_e1_12312_11111_02_000001\n
image=nothadoop/docker-image\n user=nobody\n hostname=host-id\n"
"
mounts=/var/log:/var/log:ro,/var/lib:/lib:ro,/usr/bin/cut:/usr/bin/cut:ro,/tmp:/tmp:rw\n"
- " network=bridge\n net=bridge\n"
- " cap-add=CHOWN,SETUID\n cgroup-parent=ctr-cgroup\n
detach=true\n rm=true\n"
+ " network=bridge\n cap-add=CHOWN,SETUID\n
cgroup-parent=ctr-cgroup\n detach=true\n rm=true\n"
" launch-command=bash,test_script.sh,arg1,arg2",
- "run --name=container_e1_12312_11111_02_000001 --user=nobody -d --rm
--net=bridge"
+ "run --name=container_e1_12312_11111_02_000001 --user=nobody -d --rm"
" --cgroup-parent=ctr-cgroup --cap-drop=ALL --hostname=host-id
nothadoop/docker-image bash test_script.sh arg1 arg2"));
// Test privileged container
@@ -1327,10 +1385,10 @@ namespace ContainerExecutor {
"[docker-command-execution]\n"
" docker-command=run\n name=container_e1_12312_11111_02_000001\n
image=hadoop/docker-image\n user=root\n hostname=host-id\n"
"
mounts=/var/log:/var/log:ro,/var/lib:/lib:ro,/usr/bin/cut:/usr/bin/cut:ro,/tmp:/tmp:rw\n"
- " network=bridge\n devices=/dev/test:/dev/test\n net=bridge\n
privileged=true\n"
+ " network=bridge\n devices=/dev/test:/dev/test\n
privileged=true\n"
" cap-add=CHOWN,SETUID\n cgroup-parent=ctr-cgroup\n
detach=true\n rm=true\n"
" launch-command=bash,test_script.sh,arg1,arg2",
- "run --name=container_e1_12312_11111_02_000001 -d --rm --net=bridge -v
/var/log:/var/log:ro -v /var/lib:/lib:ro"
+ "run --name=container_e1_12312_11111_02_000001 -d --rm -v
/var/log:/var/log:ro -v /var/lib:/lib:ro"
" -v /usr/bin/cut:/usr/bin/cut:ro -v /tmp:/tmp:rw
--cgroup-parent=ctr-cgroup --privileged --cap-drop=ALL "
"--cap-add=CHOWN --cap-add=SETUID --hostname=host-id
--device=/dev/test:/dev/test hadoop/docker-image "
"bash test_script.sh arg1 arg2"));
@@ -1339,10 +1397,10 @@ namespace ContainerExecutor {
"[docker-command-execution]\n"
" docker-command=run\n name=container_e1_12312_11111_02_000001\n
image=hadoop/docker-image\n user=root\n hostname=host-id\n"
"
mounts=/var/log:/var/log:ro,/var/lib:/lib:ro,/usr/bin/cut:/usr/bin/cut:ro,/tmp:/tmp:rw\n"
- " network=bridge\n devices=/dev/test:/dev/test\n net=bridge\n
privileged=true\n"
+ " network=bridge\n devices=/dev/test:/dev/test\n
privileged=true\n"
" cap-add=CHOWN,SETUID\n cgroup-parent=ctr-cgroup\n
detach=true\n rm=true\n group-add=1000,1001\n"
" launch-command=bash,test_script.sh,arg1,arg2",
- "run --name=container_e1_12312_11111_02_000001 -d --rm --net=bridge -v
/var/log:/var/log:ro -v /var/lib:/lib:ro"
+ "run --name=container_e1_12312_11111_02_000001 -d --rm -v
/var/log:/var/log:ro -v /var/lib:/lib:ro"
" -v /usr/bin/cut:/usr/bin/cut:ro -v /tmp:/tmp:rw
--cgroup-parent=ctr-cgroup --privileged --cap-drop=ALL "
"--cap-add=CHOWN --cap-add=SETUID --hostname=host-id "
"--device=/dev/test:/dev/test hadoop/docker-image bash
test_script.sh arg1 arg2"));
@@ -1350,10 +1408,9 @@ namespace ContainerExecutor {
file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
"[docker-command-execution]\n"
" docker-command=run\n name=container_e1_12312_11111_02_000001\n
image=docker-image\n user=nobody\n hostname=host-id\n"
- " network=bridge\n net=bridge\n"
- " detach=true\n rm=true\n group-add=1000,1001\n"
+ " network=bridge\n detach=true\n rm=true\n
group-add=1000,1001\n"
" launch-command=bash,test_script.sh,arg1,arg2",
- "run --name=container_e1_12312_11111_02_000001 --user=nobody -d --rm
--net=bridge --cap-drop=ALL "
+ "run --name=container_e1_12312_11111_02_000001 --user=nobody -d --rm
--cap-drop=ALL "
"--hostname=host-id --group-add 1000 --group-add 1001 "
"docker-image bash test_script.sh arg1 arg2"));
@@ -1372,7 +1429,7 @@ namespace ContainerExecutor {
"[docker-command-execution]\n"
" docker-command=run\n name=container_e1_12312_11111_02_000001\n
image=nothadoop/docker-image\n user=nobody\n hostname=host-id\n"
"
mounts=/var/log:/var/log:ro,/var/lib:/lib:ro,/usr/bin/cut:/usr/bin/cut:ro,/tmp:/tmp:rw\n"
- " network=bridge\n net=bridge\n privileged=true\n"
+ " network=bridge\n privileged=true\n"
" cap-add=CHOWN,SETUID\n cgroup-parent=ctr-cgroup\n
detach=true\n rm=true\n group-add=1000,1001\n"
" launch-command=bash,test_script.sh,arg1,arg2",
PRIVILEGED_CONTAINERS_DISABLED));
@@ -1533,10 +1590,10 @@ namespace ContainerExecutor {
"[docker-command-execution]\n"
" docker-command=run\n
name=container_e1_12312_11111_02_000001\n image=hadoop/docker-image\n
user=nobody\n hostname=host-id\n"
"
mounts=/var/log:/var/log:ro,/var/lib:/lib:ro,/usr/bin/cut:/usr/bin/cut:ro,/tmp:/tmp:rw\n"
- " network=bridge\n devices=/dev/test:/dev/test\n net=bridge\n"
+ " network=bridge\n devices=/dev/test:/dev/test\n"
" cap-add=CHOWN,SETUID\n cgroup-parent=ctr-cgroup\n
detach=true\n rm=true\n"
" launch-command=bash,test_script.sh,arg1,arg2",
- "run --name=container_e1_12312_11111_02_000001 --user=nobody -d --rm
--net=bridge -v /var/log:/var/log:ro -v /var/lib:/lib:ro"
+ "run --name=container_e1_12312_11111_02_000001 --user=nobody -d --rm
-v /var/log:/var/log:ro -v /var/lib:/lib:ro"
" -v /usr/bin/cut:/usr/bin/cut:ro -v /tmp:/tmp:rw
--cgroup-parent=ctr-cgroup --cap-drop=ALL --cap-add=CHOWN "
"--cap-add=SETUID --hostname=host-id
--device=/dev/test:/dev/test hadoop/docker-image bash"
" test_script.sh arg1 arg2"));
@@ -1544,10 +1601,9 @@ namespace ContainerExecutor {
"[docker-command-execution]\n"
" docker-command=run\n
name=container_e1_12312_11111_02_000001\n image=nothadoop/docker-image\n
user=nobody\n hostname=host-id\n"
"
mounts=/var/log:/var/log:ro,/var/lib:/lib:ro,/usr/bin/cut:/usr/bin/cut:ro,/tmp:/tmp:rw\n"
- " network=bridge\n net=bridge\n"
- " cap-add=CHOWN,SETUID\n cgroup-parent=ctr-cgroup\n
detach=true\n rm=true\n"
+ " network=bridge\n cap-add=CHOWN,SETUID\n
cgroup-parent=ctr-cgroup\n detach=true\n rm=true\n"
" launch-command=bash,test_script.sh,arg1,arg2",
- "run --name=container_e1_12312_11111_02_000001 --user=nobody -d --rm
--net=bridge"
+ "run --name=container_e1_12312_11111_02_000001 --user=nobody -d --rm"
" --cgroup-parent=ctr-cgroup --cap-drop=ALL --hostname=host-id
nothadoop/docker-image bash test_script.sh arg1 arg2"));
std::vector<std::pair<std::string, int> > bad_file_cmd_vec;
@@ -1555,7 +1611,7 @@ namespace ContainerExecutor {
"[docker-command-execution]\n"
" docker-command=run\n
name=container_e1_12312_11111_02_000001\n image=hadoop/docker-image\n
user=nobody\n hostname=host-id\n"
"
mounts=/var/log:/var/log:ro,/var/lib:/lib:ro,/usr/bin/cut:/usr/bin/cut:ro,/tmp:/tmp:rw\n"
- " network=bridge\n devices=/dev/test:/dev/test\n net=bridge\n
privileged=true\n"
+ " network=bridge\n devices=/dev/test:/dev/test\n
privileged=true\n"
" cap-add=CHOWN,SETUID\n cgroup-parent=ctr-cgroup\n
detach=true\n rm=true\n"
" launch-command=bash,test_script.sh,arg1,arg2",
static_cast<int>(PRIVILEGED_CONTAINERS_DISABLED)));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/33e0df4b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerRunCommand.java
----------------------------------------------------------------------
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerRunCommand.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerRunCommand.java
index 8dc37d4..d01c184 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerRunCommand.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerRunCommand.java
@@ -59,6 +59,10 @@ public class TestDockerRunCommand {
dockerRunCommand.setOverrideCommandWithArgs(commands);
dockerRunCommand.removeContainerOnExit();
dockerRunCommand.addTmpfsMount("/run");
+ String portsMapping = "127.0.0.1:8080:80,1234:1234,:2222";
+ for (String mapping:portsMapping.split(",")) {
+ dockerRunCommand.addPortsMapping(mapping);
+ }
assertEquals("run", StringUtils.join(",",
dockerRunCommand.getDockerCommandWithArguments()
@@ -79,7 +83,10 @@ public class TestDockerRunCommand {
.get("launch-command")));
assertEquals("/run", StringUtils.join(",",
dockerRunCommand.getDockerCommandWithArguments().get("tmpfs")));
- assertEquals(8, dockerRunCommand.getDockerCommandWithArguments().size());
+ assertEquals("127.0.0.1:8080:80,1234:1234,:2222", StringUtils.join(",",
+ dockerRunCommand.getDockerCommandWithArguments()
+ .get("ports-mapping")));
+ assertEquals(9, dockerRunCommand.getDockerCommandWithArguments().size());
}
@Test
http://git-wip-us.apache.org/repos/asf/hadoop/blob/33e0df4b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md
----------------------------------------------------------------------
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md
index 2d6f867..5620299 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md
@@ -365,6 +365,7 @@ environment variables in the application's environment:
| `YARN_CONTAINER_RUNTIME_DOCKER_IMAGE` | Names which image will be used to
launch the Docker container. Any image name that could be passed to the Docker
client's run command may be used. The image name may include a repo prefix. |
| `YARN_CONTAINER_RUNTIME_DOCKER_RUN_OVERRIDE_DISABLE` | Controls whether the
Docker container's default command is overridden. When set to true, the Docker
container's command will be "bash _path\_to\_launch\_script_". When unset or
set to false, the Docker container's default command is used. |
| `YARN_CONTAINER_RUNTIME_DOCKER_CONTAINER_NETWORK` | Sets the network type to
be used by the Docker container. It must be a valid value as determined by the
yarn.nodemanager.runtime.linux.docker.allowed-container-networks property. |
+| `YARN_CONTAINER_RUNTIME_DOCKER_PORTS_MAPPING` | Allows a user to specify
ports mapping for the bridge network Docker container. The value of the
environment variable should be a comma-separated list of ports mapping. It's
the same to "-p" option for the Docker run command. If the value is empty, "-P"
will be added. |
| `YARN_CONTAINER_RUNTIME_DOCKER_CONTAINER_PID_NAMESPACE` | Controls which PID
namespace will be used by the Docker container. By default, each Docker
container has its own PID namespace. To share the namespace of the host, the
yarn.nodemanager.runtime.linux.docker.host-pid-namespace.allowed property must
be set to true. If the host PID namespace is allowed and this environment
variable is set to host, the Docker container will share the host's PID
namespace. No other value is allowed. |
| `YARN_CONTAINER_RUNTIME_DOCKER_RUN_PRIVILEGED_CONTAINER` | Controls whether
the Docker container is a privileged container. In order to use privileged
containers, the
yarn.nodemanager.runtime.linux.docker.privileged-containers.allowed property
must be set to true, and the application owner must appear in the value of the
yarn.nodemanager.runtime.linux.docker.privileged-containers.acl property. If
this environment variable is set to true, a privileged Docker container will be
used if allowed. No other value is allowed, so the environment variable should
be left unset rather than setting it to false. |
| `YARN_CONTAINER_RUNTIME_DOCKER_MOUNTS` | Adds additional volume mounts to
the Docker container. The value of the environment variable should be a
comma-separated list of mounts. All such mounts must be given as
`source:dest[:mode]` and the mode must be "ro" (read-only) or "rw" (read-write)
to specify the type of access being requested. If neither is specified,
read-write will be assumed. The mode may include a bind propagation option. In
that case, the mode should either be of the form `[option]`, `rw+[option]`, or
`ro+[option]`. Valid bind propagation options are shared, rshared, slave,
rslave, private, and rprivate. The requested mounts will be validated by
container-executor based on the values set in container-executor.cfg for
`docker.allowed.ro-mounts` and `docker.allowed.rw-mounts`. |
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
