YARN-4266. Allow users to enter containers as UID:GID pair instead of by username. Contributed by luhuichun, Zhankun Tang, and Eric Badger.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/bfd1a72b Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/bfd1a72b Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/bfd1a72b Branch: refs/heads/HDFS-7240 Commit: bfd1a72ba8fbb06da73fede2a85e0b544d6ab43f Parents: e5e1851 Author: Jason Lowe <[email protected]> Authored: Thu Sep 21 17:41:34 2017 -0500 Committer: Jason Lowe <[email protected]> Committed: Thu Sep 21 17:41:34 2017 -0500 ---------------------------------------------------------------------- .../hadoop/yarn/conf/YarnConfiguration.java | 21 ++++++ .../src/main/resources/yarn-default.xml | 19 +++++ .../runtime/DockerLinuxContainerRuntime.java | 76 ++++++++++++++++++- .../linux/runtime/docker/DockerRunCommand.java | 7 ++ .../impl/container-executor.c | 4 + .../runtime/TestDockerContainerRuntime.java | 79 +++++++++++++++++++- 6 files changed, 204 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/bfd1a72b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 114453f..f58833c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -1587,6 +1587,27 @@ public class YarnConfiguration extends Configuration { public static final boolean DEFAULT_NM_DOCKER_ALLOW_PRIVILEGED_CONTAINERS = false; + /** enable user remapping. */ + public static final String NM_DOCKER_ENABLE_USER_REMAPPING = + DOCKER_CONTAINER_RUNTIME_PREFIX + "enable-userremapping.allowed"; + + /** Set enable user remapping as false by default. */ + public static final boolean DEFAULT_NM_DOCKER_ENABLE_USER_REMAPPING = false; + + /** lower limit for acceptable uids of user remapped user. */ + public static final String NM_DOCKER_USER_REMAPPING_UID_THRESHOLD = + DOCKER_CONTAINER_RUNTIME_PREFIX + "userremapping-uid-threshold"; + + /** Set user remapping lower uid limit to 1 by default. */ + public static final int DEFAULT_NM_DOCKER_USER_REMAPPING_UID_THRESHOLD = 1; + + /** lower limit for acceptable gids of user remapped user. */ + public static final String NM_DOCKER_USER_REMAPPING_GID_THRESHOLD = + DOCKER_CONTAINER_RUNTIME_PREFIX + "userremapping-gid-threshold"; + + /** Set user remapping lower gid limit to 1 by default. */ + public static final int DEFAULT_NM_DOCKER_USER_REMAPPING_GID_THRESHOLD = 1; + /** ACL list for users allowed to run privileged containers. */ public static final String NM_DOCKER_PRIVILEGED_CONTAINERS_ACL = DOCKER_CONTAINER_RUNTIME_PREFIX + "privileged-containers.acl"; http://git-wip-us.apache.org/repos/asf/hadoop/blob/bfd1a72b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 0440458..8453dc7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -1643,6 +1643,25 @@ </property> <property> + <description>Property to enable docker user remapping</description> + <name>yarn.nodemanager.runtime.linux.docker.enable-userremapping.allowed</name> + <value>false</value> + </property> + + <property> + <description>lower limit for acceptable uids of user remapped user</description> + <name>yarn.nodemanager.runtime.linux.docker.userremapping-uid-threshold</name> + <value>1</value> + </property> + + + <property> + <description>lower limit for acceptable gids of user remapped user</description> + <name>yarn.nodemanager.runtime.linux.docker.userremapping-gid-threshold</name> + <value>1</value> + </property> + + <property> <description>The mode in which the Java Container Sandbox should run detailed by the JavaSandboxLinuxContainerRuntime.</description> <name>yarn.nodemanager.runtime.linux.sandbox-mode</name> http://git-wip-us.apache.org/repos/asf/hadoop/blob/bfd1a72b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java index 6491a99..b8d9b0a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java @@ -30,6 +30,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.registry.client.binding.RegistryPathUtils; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.authorize.AccessControlList; +import org.apache.hadoop.util.Shell; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor; @@ -164,6 +165,9 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime { public static final String ENV_DOCKER_CONTAINER_RUN_PRIVILEGED_CONTAINER = "YARN_CONTAINER_RUNTIME_DOCKER_RUN_PRIVILEGED_CONTAINER"; @InterfaceAudience.Private + public static final String ENV_DOCKER_CONTAINER_RUN_ENABLE_USER_REMAPPING = + "YARN_CONTAINER_RUNTIME_DOCKER_RUN_ENABLE_USER_REMAPPING"; + @InterfaceAudience.Private public static final String ENV_DOCKER_CONTAINER_LOCAL_RESOURCE_MOUNTS = "YARN_CONTAINER_RUNTIME_DOCKER_LOCAL_RESOURCE_MOUNTS"; @@ -175,6 +179,9 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime { private String cgroupsRootDirectory; private CGroupsHandler cGroupsHandler; private AccessControlList privilegedContainersAcl; + private boolean enableUserReMapping; + private int userRemappingUidThreshold; + private int userRemappingGidThreshold; /** * Return whether the given environment variables indicate that the operation @@ -260,6 +267,18 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime { privilegedContainersAcl = new AccessControlList(conf.getTrimmed( YarnConfiguration.NM_DOCKER_PRIVILEGED_CONTAINERS_ACL, YarnConfiguration.DEFAULT_NM_DOCKER_PRIVILEGED_CONTAINERS_ACL)); + + enableUserReMapping = conf.getBoolean( + YarnConfiguration.NM_DOCKER_ENABLE_USER_REMAPPING, + YarnConfiguration.DEFAULT_NM_DOCKER_ENABLE_USER_REMAPPING); + + userRemappingUidThreshold = conf.getInt( + YarnConfiguration.NM_DOCKER_USER_REMAPPING_UID_THRESHOLD, + YarnConfiguration.DEFAULT_NM_DOCKER_USER_REMAPPING_UID_THRESHOLD); + + userRemappingGidThreshold = conf.getInt( + YarnConfiguration.NM_DOCKER_USER_REMAPPING_GID_THRESHOLD, + YarnConfiguration.DEFAULT_NM_DOCKER_USER_REMAPPING_GID_THRESHOLD); } @Override @@ -436,6 +455,34 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime { "resource: " + mount); } + private String getUserIdInfo(String userName) + throws ContainerExecutionException { + String id = ""; + Shell.ShellCommandExecutor shexec = new Shell.ShellCommandExecutor( + new String[]{"id", "-u", userName}); + try { + shexec.execute(); + id = shexec.getOutput().replaceAll("[^0-9]", ""); + } catch (Exception e) { + throw new ContainerExecutionException(e); + } + return id; + } + + private String[] getGroupIdInfo(String userName) + throws ContainerExecutionException { + String[] id = null; + Shell.ShellCommandExecutor shexec = new Shell.ShellCommandExecutor( + new String[]{"id", "-G", userName}); + try { + shexec.execute(); + id = shexec.getOutput().replace("\n", "").split(" "); + } catch (Exception e) { + throw new ContainerExecutionException(e); + } + return id; + } + @Override public void launchContainer(ContainerRuntimeContext ctx) throws ContainerExecutionException { @@ -458,7 +505,30 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime { String containerIdStr = container.getContainerId().toString(); String runAsUser = ctx.getExecutionAttribute(RUN_AS_USER); + String dockerRunAsUser = runAsUser; Path containerWorkDir = ctx.getExecutionAttribute(CONTAINER_WORK_DIR); + String[] groups = null; + + if (enableUserReMapping) { + String uid = getUserIdInfo(runAsUser); + groups = getGroupIdInfo(runAsUser); + String gid = groups[0]; + if(Integer.parseInt(uid) < userRemappingUidThreshold) { + String message = "uid: " + uid + " below threshold: " + + userRemappingUidThreshold; + throw new ContainerExecutionException(message); + } + for(int i = 0; i < groups.length; i++) { + String group = groups[i]; + if (Integer.parseInt(group) < userRemappingGidThreshold) { + String message = "gid: " + group + + " below threshold: " + userRemappingGidThreshold; + throw new ContainerExecutionException(message); + } + } + dockerRunAsUser = uid + ":" + gid; + } + //List<String> -> stored as List -> fetched/converted to List<String> //we can't do better here thanks to type-erasure @SuppressWarnings("unchecked") @@ -481,7 +551,7 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime { @SuppressWarnings("unchecked") DockerRunCommand runCommand = new DockerRunCommand(containerIdStr, - runAsUser, imageName) + dockerRunAsUser, imageName) .detachOnRun() .setContainerWorkDir(containerWorkDir.toString()) .setNetworkType(network); @@ -542,6 +612,10 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime { runCommand.setOverrideCommandWithArgs(overrideCommands); } + if(enableUserReMapping) { + runCommand.groupAdd(groups); + } + String commandFile = dockerClient.writeCommandToTempFile(runCommand, containerIdStr); PrivilegedOperation launchOp = buildLaunchOp(ctx, http://git-wip-us.apache.org/repos/asf/hadoop/blob/bfd1a72b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java index b645754..1e1e6e8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java @@ -114,6 +114,13 @@ public class DockerRunCommand extends DockerCommand { return this; } + public DockerRunCommand groupAdd(String[] groups) { + for(int i = 0; i < groups.length; i++) { + super.addCommandArguments("--group-add " + groups[i]); + } + return this; + } + public DockerRunCommand setOverrideCommandWithArgs( List<String> overrideCommandWithArgs) { this.overrrideCommandWithArgs = overrideCommandWithArgs; http://git-wip-us.apache.org/repos/asf/hadoop/blob/bfd1a72b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c index 560ec18..e8bf564 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c @@ -1258,6 +1258,7 @@ char* sanitize_docker_command(const char *line) { {"device", required_argument, 0, 'i' }, {"detach", required_argument, 0, 't' }, {"format", required_argument, 0, 'f' }, + {"group-add", required_argument, 0, 'x' }, {0, 0, 0, 0} }; @@ -1357,6 +1358,9 @@ char* sanitize_docker_command(const char *line) { strcat(output, optarg); strcat(output, " "); break; + case 'x': + quote_and_append_arg(&output, &output_size, "--group-add ", optarg); + break; default: fprintf(LOGFILE, "Unknown option in docker command, character %d %c, optionindex = %d\n", c, c, optind); fflush(LOGFILE); http://git-wip-us.apache.org/repos/asf/hadoop/blob/bfd1a72b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java index 0e314bf..e548790 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java @@ -24,6 +24,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.fs.Path; import org.apache.hadoop.registry.client.binding.RegistryPathUtils; +import org.apache.hadoop.util.Shell; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.conf.YarnConfiguration; @@ -77,6 +78,7 @@ public class TestDockerContainerRuntime { private ContainerLaunchContext context; private HashMap<String, String> env; private String image; + private String uidGidPair; private String runAsUser; private String user; private String appId; @@ -126,6 +128,7 @@ public class TestDockerContainerRuntime { when(context.getEnvironment()).thenReturn(env); when(container.getUser()).thenReturn(submittingUser); + uidGidPair = ""; runAsUser = "run_as_user"; user = "user"; appId = "app_id"; @@ -230,7 +233,6 @@ public class TestDockerContainerRuntime { Assert.assertEquals(13, args.size()); //verify arguments - Assert.assertEquals(runAsUser, args.get(0)); Assert.assertEquals(user, args.get(1)); Assert.assertEquals(Integer.toString(PrivilegedOperation.RunAsUserCommand .LAUNCH_DOCKER_CONTAINER.getValue()), args.get(2)); @@ -322,6 +324,81 @@ public class TestDockerContainerRuntime { } @Test + public void testContainerLaunchWithUserRemapping() + throws ContainerExecutionException, PrivilegedOperationException, + IOException { + conf.setBoolean(YarnConfiguration.NM_DOCKER_ENABLE_USER_REMAPPING, + true); + Shell.ShellCommandExecutor shexec = new Shell.ShellCommandExecutor( + new String[]{"whoami"}); + shexec.execute(); + // get rid of newline at the end + runAsUser = shexec.getOutput().replaceAll("\n$", ""); + builder.setExecutionAttribute(RUN_AS_USER, runAsUser); + + DockerLinuxContainerRuntime runtime = new DockerLinuxContainerRuntime( + mockExecutor, mockCGroupsHandler); + runtime.initialize(conf); + runtime.launchContainer(builder.build()); + + PrivilegedOperation op = capturePrivilegedOperationAndVerifyArgs(); + List<String> args = op.getArguments(); + String dockerCommandFile = args.get(11); + + String uid = ""; + String gid = ""; + Shell.ShellCommandExecutor shexec1 = new Shell.ShellCommandExecutor( + new String[]{"id", "-u", runAsUser}); + Shell.ShellCommandExecutor shexec2 = new Shell.ShellCommandExecutor( + new String[]{"id", "-g", runAsUser}); + try { + shexec1.execute(); + // get rid of newline at the end + uid = shexec1.getOutput().replaceAll("\n$", ""); + } catch (Exception e) { + LOG.info("Could not run id -u command: " + e); + } + try { + shexec2.execute(); + // get rid of newline at the end + gid = shexec2.getOutput().replaceAll("\n$", ""); + } catch (Exception e) { + LOG.info("Could not run id -g command: " + e); + } + uidGidPair = uid + ":" + gid; + + //This is the expected docker invocation for this case + StringBuffer expectedCommandTemplate = new StringBuffer("run --name=%1$s ") + .append("--user=%2$s -d ") + .append("--workdir=%3$s ") + .append("--net=host ") + .append("--hostname=" + defaultHostname + " ") + .append(getExpectedTestCapabilitiesArgumentString()) + .append(getExpectedCGroupsMountString()) + .append("-v %4$s:%4$s ") + .append("-v %5$s:%5$s ") + .append("-v %6$s:%6$s ") + .append("-v %7$s:%7$s ") + .append("-v %8$s:%8$s ") + .append("(--group-add \\d+ )*") + .append("%9$s ") + .append("bash %10$s/launch_container.sh"); + + String expectedCommand = String + .format(expectedCommandTemplate.toString(), containerId, uidGidPair, + containerWorkDir, containerLocalDirs.get(0), filecacheDirs.get(0), + containerWorkDir, containerLogDirs.get(0), userLocalDirs.get(0), + image, containerWorkDir); + + List<String> dockerCommands = Files.readAllLines( + Paths.get(dockerCommandFile), Charset.forName("UTF-8")); + + Assert.assertEquals(1, dockerCommands.size()); + //Assert.assertEquals(expectedCommand, dockerCommands.get(0)); + Assert.assertTrue(dockerCommands.get(0).matches(expectedCommand)); + } + + @Test public void testAllowedNetworksConfiguration() throws ContainerExecutionException { //the default network configuration should cause --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
