Repository: incubator-myriad Updated Branches: refs/heads/master 5118cff3e -> fe493af32
Created better cgroup support please see cgroups.md for documentation. JIRA: [Myriad-192] https://issues.apache.org/jira/browse/MYRIAD-192 Pull Request: Closes #69 Author: DarinJ <dar...@apache.org> Project: http://git-wip-us.apache.org/repos/asf/incubator-myriad/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-myriad/commit/fe493af3 Tree: http://git-wip-us.apache.org/repos/asf/incubator-myriad/tree/fe493af3 Diff: http://git-wip-us.apache.org/repos/asf/incubator-myriad/diff/fe493af3 Branch: refs/heads/master Commit: fe493af32eeee9e069f9c228c8847545d31a47be Parents: 5118cff Author: darinj <darinj.w...@gmail.com> Authored: Fri May 6 00:02:57 2016 -0400 Committer: darinj <dar...@apache.org> Committed: Wed May 11 23:56:19 2016 -0400 ---------------------------------------------------------------------- docs/cgroups.md | 49 +++++++++----- .../configuration/MyriadConfiguration.java | 8 ++- .../MyriadExecutorConfiguration.java | 1 + .../scheduler/DownloadNMExecutorCLGenImpl.java | 22 +++---- .../myriad/scheduler/NMExecutorCLGenImpl.java | 67 ++++++++++---------- .../scheduler/ServiceCommandLineGenerator.java | 2 +- .../scheduler/TestServiceCommandLine.java | 13 ++-- 7 files changed, 90 insertions(+), 72 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/fe493af3/docs/cgroups.md ---------------------------------------------------------------------- diff --git a/docs/cgroups.md b/docs/cgroups.md index 47228bb..8694db8 100644 --- a/docs/cgroups.md +++ b/docs/cgroups.md @@ -30,18 +30,35 @@ Enabling Cgroups for YARN NodeManager involves: * Modifying the $YARN_HOME/etc/hadoop/myriad-config-default.yml file. * Modifying the $YARN_HOME/etc/hadoop/yarn-site.xml file. +### Modify container-executor.cfg +``` +yarn.nodemanager.linux-container-executor.group=yarn #should match yarn.nodemanager.linux-container-executor.group in yarn-site.xml +banned.users= +min.user.id=1000 +``` +### Verify Permissions + +The paths to container-executor.cfg and container-executor must be owned and writable only by root. The container-executor +should have user-ownership by root and group ownership by the user running YARN (often yarn or hduser), which should match the +yarn.nodemanager.linux-container-executor.group in yarn-site.xml and yarn.nodemanager.linux-container-executor.group in +container-executor.cfg. Further the permission of container-executor should be r-Sr-s---. +``` +chmod 6050 container-executor +``` +If using remote distribution be sure to use the -p option of tar (as root) to perserve the suid bit. + ### Modify Myriad-Config-default.yml ### Modify the $YARN_HOME/etc/hadoop/myriad-config-default.yml file by adding the following content: ``` ... +frameworkSuperUser: root # Must be root or have passwordless sudo. nodemanager: -cgroups: true + cgroupPath: /path/to/cgroup # default is /sys/fs/cgroup ... ``` - ### Modify yarn-site.yml Modify the `$YARN_HOME/etc/hadoop/yarn-site.xml` file by adding the following content: @@ -50,33 +67,33 @@ Modify the `$YARN_HOME/etc/hadoop/yarn-site.xml` file by adding the following co <property> <description>who will execute(launch) the containers.</description> <name>yarn.nodemanager.container-executor.class</name> -<value>${yarn.nodemanager.container-executor.class}</value> +<value>org.apache.hadoop.yarn.server.nodemanager.LinuxContainerExecutor</value> </property> <property> <description>The class which should help the LCE handle resources.</description> <name>yarn.nodemanager.linux-container-executor.resources-handler.class</name> -<value>${yarn.nodemanager.linux-container-executor.resources-handler.class}</value> +<value>org.apache.hadoop.yarn.server.nodemanager.util.CgroupsLCEResourcesHandler</value> </property> <property> -<name>yarn.nodemanager.linux-container-executor.cgroups.hierarchy</name> -<value>${yarn.nodemanager.linux-container-executor.cgroups.hierarchy}</value> -</property> -<property> -<name>yarn.nodemanager.linux-container-executor.cgroups.mount</name> -<value>${yarn.nodemanager.linux-container-executor.cgroups.mount}</value> +<name>yarn.nodemanager.linux-container-executor.group</name> +<value>yarn</value> </property> <property> -<name>yarn.nodemanager.linux-container-executor.cgroups.mount-path</name> -<value>${yarn.nodemanager.linux-container-executor.cgroups.mount-path}</value> +<name>yarn.nodemanager.linux-container-executor.path</name> +<value>${yarn.home}/bin/container-executor</value> </property> + +<!-- Optional parameters, usually unnecessary <property> -<name>yarn.nodemanager.linux-container-executor.group</name> -<value>${yarn.nodemanager.linux-container-executor.group}</value> +<name>yarn.nodemanager.linux-container-executor.cgroups.mount</name> +<value>true</value> </property> <property> -<name>yarn.nodemanager.linux-container-executor.path</name> -<value>${yarn.home}/bin/container-executor</value> +<name>yarn.nodemanager.linux-container-executor.cgroups.mount-path</name> +<value>/sys/fs/cgroup</value> +<description>/sys/fs/cgroup and /cgroup are most common values</description> </property> +--> ``` --- <sub> http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/fe493af3/myriad-scheduler/src/main/java/org/apache/myriad/configuration/MyriadConfiguration.java ---------------------------------------------------------------------- diff --git a/myriad-scheduler/src/main/java/org/apache/myriad/configuration/MyriadConfiguration.java b/myriad-scheduler/src/main/java/org/apache/myriad/configuration/MyriadConfiguration.java index f65bb9a..3de72a6 100644 --- a/myriad-scheduler/src/main/java/org/apache/myriad/configuration/MyriadConfiguration.java +++ b/myriad-scheduler/src/main/java/org/apache/myriad/configuration/MyriadConfiguration.java @@ -50,7 +50,7 @@ import org.hibernate.validator.constraints.NotEmpty; * jvmMaxMemoryMB: 1024 * user: hduser * cpus: 0.2 - * cgroups: false + * cgroupPath: /sys/fs/cgroup * executor: * jvmMaxMemoryMB: 256 * path: file://localhost/usr/local/libexec/mesos/myriad-executor-runnable-0.1.0.jar @@ -185,6 +185,9 @@ public class MyriadConfiguration { @JsonProperty private String servedBinaryPath; + @JsonProperty + private String cgroupPath; + public MyriadConfiguration() { } @@ -291,4 +294,7 @@ public class MyriadConfiguration { return Optional.fromNullable(servedBinaryPath); } + public String getCGroupPath() { + return cgroupPath == null ? "/sys/fs/cgroup" : cgroupPath; + } } http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/fe493af3/myriad-scheduler/src/main/java/org/apache/myriad/configuration/MyriadExecutorConfiguration.java ---------------------------------------------------------------------- diff --git a/myriad-scheduler/src/main/java/org/apache/myriad/configuration/MyriadExecutorConfiguration.java b/myriad-scheduler/src/main/java/org/apache/myriad/configuration/MyriadExecutorConfiguration.java index ea98ef7..1096e16 100644 --- a/myriad-scheduler/src/main/java/org/apache/myriad/configuration/MyriadExecutorConfiguration.java +++ b/myriad-scheduler/src/main/java/org/apache/myriad/configuration/MyriadExecutorConfiguration.java @@ -77,4 +77,5 @@ public class MyriadExecutorConfiguration { public Optional<String> getJvmUri() { return Optional.fromNullable(jvmUri); } + } http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/fe493af3/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/DownloadNMExecutorCLGenImpl.java ---------------------------------------------------------------------- diff --git a/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/DownloadNMExecutorCLGenImpl.java b/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/DownloadNMExecutorCLGenImpl.java index 67fd1ae..74deda3 100644 --- a/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/DownloadNMExecutorCLGenImpl.java +++ b/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/DownloadNMExecutorCLGenImpl.java @@ -44,12 +44,11 @@ public class DownloadNMExecutorCLGenImpl extends NMExecutorCLGenImpl { public String generateCommandLine(ServiceResourceProfile profile, Ports ports) { StringBuilder cmdLine = new StringBuilder(); LOGGER.info("Using remote distribution"); - generateEnvironment(profile, (NMPorts) ports); appendDistroExtractionCommands(cmdLine); appendCgroupsCmds(cmdLine); appendYarnHomeExport(cmdLine); - appendUser(cmdLine); + appendUserSudo(cmdLine); appendEnvForNM(cmdLine); cmdLine.append(YARN_NM_CMD); return cmdLine.toString(); @@ -65,21 +64,16 @@ public class DownloadNMExecutorCLGenImpl extends NMExecutorCLGenImpl { //TODO(DarinJ) support other compression, as this is a temp fix for Mesos 1760 may not get to it. //Extract tarball keeping permissions, necessary to keep HADOOP_HOME/bin/container-executor suidbit set. - cmdLine.append("sudo tar -zxpf ").append(getFileName(nodeManagerUri)); - - //We need the current directory to be writable by frameworkUser for capsuleExecutor to create directories. - //Best to simply give owenership to the user running the executor but we don't want to use -R as this - //will silently remove the suid bit on container executor. - cmdLine.append(" && sudo chown ").append(cfg.getFrameworkUser().get()).append(" ."); - + appendSudo(cmdLine); + cmdLine.append("tar -zxpf ").append(getFileName(nodeManagerUri)); //Place the hadoop config where in the HADOOP_CONF_DIR where it will be read by the NodeManager //The url for the resource manager config is: http(s)://hostname:port/conf so fetcher.cpp downloads the //config file to conf, It's an xml file with the parameters of yarn-site.xml, core-site.xml and hdfs.xml. - cmdLine.append(" && cp conf ").append(cfg.getYarnEnvironment().get("YARN_HOME")).append("/etc/hadoop/yarn-site.xml;"); - } - - protected void appendUser(StringBuilder cmdLine) { - cmdLine.append(" sudo -E -u ").append(cfg.getFrameworkUser().get()).append(" -H"); + cmdLine.append(" && "); + appendSudo(cmdLine); + cmdLine.append(" cp conf "); + cmdLine.append(cfg.getYarnEnvironment().get("YARN_HOME")); + cmdLine.append("/etc/hadoop/yarn-site.xml;"); } private static String getFileName(String uri) { http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/fe493af3/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/NMExecutorCLGenImpl.java ---------------------------------------------------------------------- diff --git a/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/NMExecutorCLGenImpl.java b/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/NMExecutorCLGenImpl.java index 19eda34..2700488 100644 --- a/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/NMExecutorCLGenImpl.java +++ b/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/NMExecutorCLGenImpl.java @@ -38,27 +38,10 @@ public class NMExecutorCLGenImpl implements ExecutorCommandLineGenerator { public static final String KEY_YARN_RM_HOSTNAME = "yarn.resourcemanager.hostname"; /** - * YARN container executor class. - */ - public static final String KEY_YARN_NM_CONTAINER_EXECUTOR_CLASS = "yarn.nodemanager.container-executor.class"; - // TODO (mohit): Should it be configurable ? - public static final String VAL_YARN_NM_CONTAINER_EXECUTOR_CLASS = - "org.apache.hadoop.yarn.server.nodemanager.LinuxContainerExecutor"; - public static final String DEFAULT_YARN_NM_CONTAINER_EXECUTOR_CLASS = - "org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor"; - /** * YARN class to help handle LCE resources */ - public static final String KEY_YARN_NM_LCE_RH_CLASS = "yarn.nodemanager.linux-container-executor.resources-handler.class"; // TODO (mohit): Should it be configurable ? - public static final String VAL_YARN_NM_LCE_RH_CLASS = "org.apache.hadoop.yarn.server.nodemanager.util.CgroupsLCEResourcesHandler"; - public static final String KEY_YARN_NM_LCE_CGROUPS_HIERARCHY = "yarn.nodemanager.linux-container-executor.cgroups.hierarchy"; - public static final String VAL_YARN_NM_LCE_CGROUPS_HIERARCHY = "mesos/$TASK_DIR"; - public static final String KEY_YARN_NM_LCE_CGROUPS_MOUNT = "yarn.nodemanager.linux-container-executor.cgroups.mount"; - public static final String KEY_YARN_NM_LCE_CGROUPS_MOUNT_PATH = "yarn.nodemanager.linux-container-executor.cgroups.mount-path"; - public static final String VAL_YARN_NM_LCE_CGROUPS_MOUNT_PATH = "/sys/fs/cgroup"; - public static final String KEY_YARN_NM_LCE_GROUP = "yarn.nodemanager.linux-container-executor.group"; - public static final String KEY_YARN_NM_LCE_PATH = "yarn.nodemanager.linux-container-executor.path"; + public static final String KEY_YARN_NM_LCE_CGROUPS_HIERARCHY = "yarn.nodemanager.linux-container-executor.cgroups.hierachy"; public static final String KEY_YARN_HOME = "yarn.home"; public static final String KEY_NM_RESOURCE_CPU_VCORES = "nodemanager.resource.cpu-vcores"; public static final String KEY_NM_RESOURCE_MEM_MB = "nodemanager.resource.memory-mb"; @@ -73,6 +56,7 @@ public class NMExecutorCLGenImpl implements ExecutorCommandLineGenerator { private Map<String, String> environment = new HashMap<>(); protected MyriadConfiguration cfg; + protected YarnConfiguration conf = new YarnConfiguration(); public NMExecutorCLGenImpl(MyriadConfiguration cfg) { this.cfg = cfg; @@ -103,21 +87,10 @@ public class NMExecutorCLGenImpl implements ExecutorCommandLineGenerator { } if (cfg.getNodeManagerConfiguration().getCgroups().or(Boolean.FALSE)) { - addYarnNodemanagerOpt(KEY_YARN_NM_CONTAINER_EXECUTOR_CLASS, VAL_YARN_NM_CONTAINER_EXECUTOR_CLASS); - addYarnNodemanagerOpt(KEY_YARN_NM_LCE_RH_CLASS, VAL_YARN_NM_LCE_RH_CLASS); - - // TODO: Configure hierarchy - addYarnNodemanagerOpt(KEY_YARN_NM_LCE_CGROUPS_HIERARCHY, VAL_YARN_NM_LCE_CGROUPS_HIERARCHY); - addYarnNodemanagerOpt(KEY_YARN_NM_LCE_CGROUPS_MOUNT, "true"); - // TODO: Make it configurable - addYarnNodemanagerOpt(KEY_YARN_NM_LCE_CGROUPS_MOUNT_PATH, VAL_YARN_NM_LCE_CGROUPS_MOUNT_PATH); - addYarnNodemanagerOpt(KEY_YARN_NM_LCE_GROUP, "root"); + addYarnNodemanagerOpt(KEY_YARN_NM_LCE_CGROUPS_HIERARCHY, "mesos/$TASK_DIR"); if (environment.containsKey("YARN_HOME")) { addYarnNodemanagerOpt(KEY_YARN_HOME, environment.get("YARN_HOME")); } - } else { - // Otherwise configure to use Default - addYarnNodemanagerOpt(KEY_YARN_NM_CONTAINER_EXECUTOR_CLASS, DEFAULT_YARN_NM_CONTAINER_EXECUTOR_CLASS); } addYarnNodemanagerOpt(KEY_NM_RESOURCE_CPU_VCORES, Integer.toString(profile.getCpus().intValue())); addYarnNodemanagerOpt(KEY_NM_RESOURCE_MEM_MB, Integer.toString(profile.getMemory().intValue())); @@ -135,15 +108,40 @@ public class NMExecutorCLGenImpl implements ExecutorCommandLineGenerator { } protected void appendCgroupsCmds(StringBuilder cmdLine) { - if (cfg.getNodeManagerConfiguration().getCgroups().or(Boolean.FALSE)) { - cmdLine.append(" export TASK_DIR=`basename $PWD`;"); - cmdLine.append(" chmod +x /sys/fs/cgroup/cpu/mesos/$TASK_DIR;"); + if (cfg.getFrameworkSuperUser().isPresent()) { + cmdLine.append(" export TASK_DIR=`basename $PWD`&&"); + appendSudo(cmdLine); + //The container executor script expects mount-path to exist and owned by the yarn user + //See: https://hadoop.apache.org/docs/stable/hadoop-yarn/hadoop-yarn-site/NodeManagerCgroups.html + //If YARN ever moves to cgroup/mem it will be necessary to add a mem version. + appendSudo(cmdLine); + cmdLine.append("chown " + cfg.getFrameworkUser().get() + " "); + cmdLine.append(cfg.getCGroupPath()); + cmdLine.append("/cpu/mesos/$TASK_DIR &&"); + } else { + LOGGER.info("frameworkSuperUser not enabled ignoring cgroup configuration"); } } protected void appendYarnHomeExport(StringBuilder cmdLine) { if (environment.containsKey("YARN_HOME")) { - cmdLine.append(" export YARN_HOME=" + environment.get("YARN_HOME") + ";"); + cmdLine.append(" export YARN_HOME="); + cmdLine.append(environment.get("YARN_HOME")); + cmdLine.append(";"); + } + } + + protected void appendSudo(StringBuilder cmdLine) { + if (cfg.getFrameworkSuperUser().isPresent()) { + cmdLine.append(" sudo "); + } + } + + protected void appendUserSudo(StringBuilder cmdLine) { + if (cfg.getFrameworkSuperUser().isPresent()) { + cmdLine.append(" sudo -E -u "); + cmdLine.append(cfg.getFrameworkUser().get()); + cmdLine.append(" -H "); } } @@ -159,7 +157,6 @@ public class NMExecutorCLGenImpl implements ExecutorCommandLineGenerator { @Override public String getConfigurationUrl() { - YarnConfiguration conf = new YarnConfiguration(); String httpPolicy = conf.get(TaskFactory.YARN_HTTP_POLICY); if (httpPolicy != null && httpPolicy.equals(TaskFactory.YARN_HTTP_POLICY_HTTPS_ONLY)) { String address = conf.get(TaskFactory.YARN_RESOURCEMANAGER_WEBAPP_HTTPS_ADDRESS); http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/fe493af3/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/ServiceCommandLineGenerator.java ---------------------------------------------------------------------- diff --git a/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/ServiceCommandLineGenerator.java b/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/ServiceCommandLineGenerator.java index 6fd8872..8765226 100644 --- a/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/ServiceCommandLineGenerator.java +++ b/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/ServiceCommandLineGenerator.java @@ -33,7 +33,7 @@ public class ServiceCommandLineGenerator extends DownloadNMExecutorCLGenImpl { public String generateCommandLine(ServiceResourceProfile profile, Ports ports) { StringBuilder strB = new StringBuilder(); appendDistroExtractionCommands(strB); - appendUser(strB); + appendUserSudo(strB); return strB.toString(); } http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/fe493af3/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/TestServiceCommandLine.java ---------------------------------------------------------------------- diff --git a/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/TestServiceCommandLine.java b/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/TestServiceCommandLine.java index ab7e7d9..e49c19c 100644 --- a/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/TestServiceCommandLine.java +++ b/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/TestServiceCommandLine.java @@ -37,12 +37,12 @@ public class TestServiceCommandLine { static MyriadConfiguration cfg; static String toJHSCompare = - "echo \"sudo tar -zxpf hadoop-2.7.0.tar.gz && sudo chown hduser . && cp conf /usr/local/hadoop/etc/hadoop/yarn-site.xml; " + - "sudo -E -u hduser -H $YARN_HOME/bin/mapred historyserver\";sudo tar -zxpf hadoop-2.5.0.tar.gz && sudo chown hduser . && cp" + + "echo \" sudo tar -zxpf hadoop-2.7.0.tar.gz && sudo cp conf /usr/local/hadoop/etc/hadoop/yarn-site.xml; " + + "export TASK_DIR=`basename $PWD`; sudo chmod +wx /sys/fs/cgroup/cpu/mesos/$TASK_DIR;" + + "sudo -E -u hduser -H $YARN_HOME/bin/mapred historyserver\"; sudo tar -zxpf hadoop-2.5.0.tar.gz && sudo cp" + " conf /usr/local/hadoop/etc/hadoop/yarn-site.xml; sudo -E -u hduser -H $YARN_HOME/bin/mapred historyserver"; - static String toCompare = - "echo \"sudo tar -zxpf hadoop-2.7.0.tar.gz && sudo chown hduser . && cp conf /usr/local/hadoop/etc/hadoop/yarn-site.xml;"; + "echo \" sudo tar -zxpf hadoop-2.7.0.tar.gz && sudo cp conf /usr/local/hadoop/etc/hadoop/yarn-site.xml;"; @BeforeClass public static void setUpBeforeClass() throws Exception { @@ -63,6 +63,8 @@ public class TestServiceCommandLine { ServiceResourceProfile profile = new ServiceResourceProfile("jobhistory", 10.0, 15.0); CommandInfo cInfo = jhs.createCommandInfo(profile, executorCmd); + System.out.println(toJHSCompare); + System.out.println(cInfo.getValue()); assertTrue(cInfo.getValue().startsWith(toCompare)); } @@ -79,7 +81,8 @@ public class TestServiceCommandLine { NMTaskFactoryImpl nms = new NMTaskFactoryImpl(cfg, null, clGenerator); CommandInfo cInfo = nms.getCommandInfo(profile, nmPorts); - + System.out.println(toCompare); + System.out.println(cInfo.getValue()); assertTrue(cInfo.getValue().startsWith(toCompare)); }
