Repository: hadoop Updated Branches: refs/heads/branch-2 6722e17f6 -> 872b8d90a
YARN-4579. Allow DefaultContainerExecutor container log directory permissions to be configurable (rchiang via rkanter) (cherry picked from commit d7fdec1e6b465395d2faca6e404e329d20f6c3d8) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/872b8d90 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/872b8d90 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/872b8d90 Branch: refs/heads/branch-2 Commit: 872b8d90a6ab91459f0bda8a7750f957da489131 Parents: 6722e17 Author: Robert Kanter <[email protected]> Authored: Thu Feb 25 16:35:58 2016 -0800 Committer: Robert Kanter <[email protected]> Committed: Thu Feb 25 16:40:05 2016 -0800 ---------------------------------------------------------------------- hadoop-yarn-project/CHANGES.txt | 3 ++ .../hadoop/yarn/conf/YarnConfiguration.java | 7 +++++ .../src/main/resources/yarn-default.xml | 10 ++++++ .../nodemanager/DefaultContainerExecutor.java | 32 +++++++++++++++++--- .../TestDefaultContainerExecutor.java | 21 +++++++++---- 5 files changed, 62 insertions(+), 11 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/872b8d90/hadoop-yarn-project/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 07ed65e..cf4d5fd 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -794,6 +794,9 @@ Release 2.8.0 - UNRELEASED YARN-4682. AMRM client to log when AMRM token updated. (Prabhu Joseph via stevel) + YARN-4579. Allow DefaultContainerExecutor container log directory + permissions to be configurable (rchiang via rkanter) + OPTIMIZATIONS YARN-3339. TestDockerContainerExecutor should pull a single image and not http://git-wip-us.apache.org/repos/asf/hadoop/blob/872b8d90/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index ef44149..a75e321 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -764,6 +764,13 @@ public class YarnConfiguration extends Configuration { NM_PREFIX + "logaggregation.threadpool-size-max"; public static final int DEFAULT_NM_LOG_AGGREGATION_THREAD_POOL_SIZE = 100; + /** Default permissions for container logs. */ + public static final String NM_DEFAULT_CONTAINER_EXECUTOR_PREFIX = + NM_PREFIX + "default-container-executor."; + public static final String NM_DEFAULT_CONTAINER_EXECUTOR_LOG_DIRS_PERMISSIONS = + NM_DEFAULT_CONTAINER_EXECUTOR_PREFIX + "log-dirs.permissions"; + public static final String NM_DEFAULT_CONTAINER_EXECUTOR_LOG_DIRS_PERMISSIONS_DEFAULT = "710"; + public static final String NM_RESOURCEMANAGER_MINIMUM_VERSION = NM_PREFIX + "resourcemanager.minimum.version"; public static final String DEFAULT_NM_RESOURCEMANAGER_MINIMUM_VERSION = "NONE"; http://git-wip-us.apache.org/repos/asf/hadoop/blob/872b8d90/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 72c89c9..cc08802 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -1073,6 +1073,16 @@ </property> <property> + <description> + The permissions settings used for the creation of container + directories when using DefaultContainerExecutor. This follows + standard user/group/all permissions format. + </description> + <name>yarn.nodemanager.default-container-executor.log-dirs.permissions</name> + <value>710</value> + </property> + + <property> <description>Whether to enable log aggregation. Log aggregation collects each container's logs and moves these logs onto a file-system, for e.g. HDFS, after the application completes. Users can configure the http://git-wip-us.apache.org/repos/asf/hadoop/blob/872b8d90/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java index 9ccde5f..49398e4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java @@ -74,6 +74,8 @@ public class DefaultContainerExecutor extends ContainerExecutor { protected final FileContext lfs; + private String logDirPermissions = null; + public DefaultContainerExecutor() { try { this.lfs = FileContext.getLocalFSFileContext(); @@ -509,9 +511,6 @@ public class DefaultContainerExecutor extends ContainerExecutor { /** Permissions for user app dir. * $local.dir/usercache/$user/appcache/$appId */ static final short APPDIR_PERM = (short)0710; - /** Permissions for user log dir. - * $logdir/$user/$appId */ - static final short LOGDIR_PERM = (short)0710; private long getDiskFreeSpace(Path base) throws IOException { return lfs.getFsStatus(base).getRemaining(); @@ -702,7 +701,8 @@ public class DefaultContainerExecutor extends ContainerExecutor { throws IOException { boolean appLogDirStatus = false; - FsPermission appLogDirPerms = new FsPermission(LOGDIR_PERM); + FsPermission appLogDirPerms = new + FsPermission(getLogDirPermissions()); for (String rootLogDir : logDirs) { // create $log.dir/$appid Path appLogDir = new Path(rootLogDir, appId); @@ -727,7 +727,8 @@ public class DefaultContainerExecutor extends ContainerExecutor { List<String> logDirs, String user) throws IOException { boolean containerLogDirStatus = false; - FsPermission containerLogDirPerms = new FsPermission(LOGDIR_PERM); + FsPermission containerLogDirPerms = new + FsPermission(getLogDirPermissions()); for (String rootLogDir : logDirs) { // create $log.dir/$appid/$containerid Path appLogDir = new Path(rootLogDir, appId); @@ -750,6 +751,27 @@ public class DefaultContainerExecutor extends ContainerExecutor { } /** + * Return default container log directory permissions. + */ + @VisibleForTesting + public String getLogDirPermissions() { + if (this.logDirPermissions==null) { + this.logDirPermissions = getConf().get( + YarnConfiguration.NM_DEFAULT_CONTAINER_EXECUTOR_LOG_DIRS_PERMISSIONS, + YarnConfiguration.NM_DEFAULT_CONTAINER_EXECUTOR_LOG_DIRS_PERMISSIONS_DEFAULT); + } + return this.logDirPermissions; + } + + /** + * Clear the internal variable for repeatable testing. + */ + @VisibleForTesting + public void clearLogDirPermissions() { + this.logDirPermissions = null; + } + + /** * @return the list of paths of given local directories */ private static List<Path> getPaths(List<String> dirs) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/872b8d90/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDefaultContainerExecutor.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDefaultContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDefaultContainerExecutor.java index 4404a7c..5696ae8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDefaultContainerExecutor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDefaultContainerExecutor.java @@ -168,8 +168,7 @@ public class TestDefaultContainerExecutor { DefaultContainerExecutor.FILECACHE_PERM); final FsPermission appDirPerm = new FsPermission( DefaultContainerExecutor.APPDIR_PERM); - final FsPermission logDirPerm = new FsPermission( - DefaultContainerExecutor.LOGDIR_PERM); + List<String> localDirs = new ArrayList<String>(); localDirs.add(new Path(BASE_TMP_PATH, "localDirA").toString()); localDirs.add(new Path(BASE_TMP_PATH, "localDirB").toString()); @@ -181,6 +180,7 @@ public class TestDefaultContainerExecutor { conf.set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY, "077"); FileContext lfs = FileContext.getLocalFSFileContext(conf); DefaultContainerExecutor executor = new DefaultContainerExecutor(lfs); + executor.setConf(conf); executor.init(); try { @@ -208,11 +208,20 @@ public class TestDefaultContainerExecutor { Assert.assertEquals(appDirPerm, stats.getPermission()); } - executor.createAppLogDirs(appId, logDirs, user); + String[] permissionsArray = { "000", "111", "555", "710", "777" }; + + for (String perm : permissionsArray ) { + conf.set(YarnConfiguration.NM_DEFAULT_CONTAINER_EXECUTOR_LOG_DIRS_PERMISSIONS, perm); + executor.clearLogDirPermissions(); + FsPermission logDirPerm = new FsPermission( + executor.getLogDirPermissions()); + executor.createAppLogDirs(appId, logDirs, user); - for (String dir : logDirs) { - FileStatus stats = lfs.getFileStatus(new Path(dir, appId)); - Assert.assertEquals(logDirPerm, stats.getPermission()); + for (String dir : logDirs) { + FileStatus stats = lfs.getFileStatus(new Path(dir, appId)); + Assert.assertEquals(logDirPerm, stats.getPermission()); + lfs.delete(new Path(dir, appId), true); + } } } finally { deleteTmpFiles();
