MAPREDUCE-7010. Make Job History File Permissions configurable. Contributed by Gergely Novák
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5837a940 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5837a940 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5837a940 Branch: refs/heads/YARN-7055 Commit: 5837a94070271f026958aa589b121123d2e03668 Parents: 0398391 Author: Billie Rinaldi <bil...@apache.org> Authored: Mon Feb 26 14:32:46 2018 -0800 Committer: Rohith Sharma K S <rohithsharm...@apache.org> Committed: Fri Mar 2 11:08:28 2018 +0530 ---------------------------------------------------------------------- .../jobhistory/JobHistoryEventHandler.java | 4 +-- .../mapreduce/v2/jobhistory/JHAdminConfig.java | 4 +++ .../v2/jobhistory/JobHistoryUtils.java | 38 ++++++++++++++++---- .../v2/jobhistory/TestJobHistoryUtils.java | 24 +++++++++++++ .../src/main/resources/mapred-default.xml | 9 +++++ 5 files changed, 70 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/5837a940/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java index ae46129..fd93d07 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java @@ -231,8 +231,8 @@ public class JobHistoryEventHandler extends AbstractService try { doneDirPrefixPath = FileContext.getFileContext(conf).makeQualified(new Path(userDoneDirStr)); - mkdir(doneDirFS, doneDirPrefixPath, new FsPermission( - JobHistoryUtils.HISTORY_INTERMEDIATE_USER_DIR_PERMISSIONS)); + mkdir(doneDirFS, doneDirPrefixPath, JobHistoryUtils. + getConfiguredHistoryIntermediateUserDoneDirPermissions(conf)); } catch (IOException e) { LOG.error("Error creating user intermediate history done directory: [ " + doneDirPrefixPath + "]", e); http://git-wip-us.apache.org/repos/asf/hadoop/blob/5837a940/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java index 5097946..1cadf84 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java @@ -93,6 +93,10 @@ public class JHAdminConfig { **/ public static final String MR_HISTORY_INTERMEDIATE_DONE_DIR = MR_HISTORY_PREFIX + "intermediate-done-dir"; + public static final String MR_HISTORY_INTERMEDIATE_USER_DONE_DIR_PERMISSIONS = + MR_HISTORY_PREFIX + "intermediate-user-done-dir.permissions"; + public static final short + DEFAULT_MR_HISTORY_INTERMEDIATE_USER_DONE_DIR_PERMISSIONS = 0770; /** Size of the job list cache.*/ public static final String MR_HISTORY_JOBLIST_CACHE_SIZE = http://git-wip-us.apache.org/repos/asf/hadoop/blob/5837a940/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JobHistoryUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JobHistoryUtils.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JobHistoryUtils.java index 58ac1db..5160ce8 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JobHistoryUtils.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JobHistoryUtils.java @@ -38,6 +38,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.PathFilter; import org.apache.hadoop.fs.RemoteIterator; import org.apache.hadoop.fs.UnsupportedFileSystemException; +import org.apache.hadoop.fs.permission.FsAction; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.mapreduce.JobID; import org.apache.hadoop.mapreduce.MRJobConfig; @@ -89,13 +90,7 @@ public class JobHistoryUtils { */ public static final FsPermission HISTORY_INTERMEDIATE_DONE_DIR_PERMISSIONS = FsPermission.createImmutable((short) 01777); - - /** - * Permissions for the user directory under the intermediate done directory. - */ - public static final FsPermission HISTORY_INTERMEDIATE_USER_DIR_PERMISSIONS = - FsPermission.createImmutable((short) 0770); - + public static final FsPermission HISTORY_INTERMEDIATE_FILE_PERMISSIONS = FsPermission.createImmutable((short) 0770); // rwx------ @@ -208,6 +203,35 @@ public class JobHistoryUtils { } return ensurePathInDefaultFileSystem(doneDirPrefix, conf); } + + /** + * Gets the configured directory permissions for the user directories in the + * directory of the intermediate done history files. The user and the group + * both need full permissions, this is enforced by this method. + * @param conf The configuration object + * @return FsPermission of the user directories + */ + public static FsPermission + getConfiguredHistoryIntermediateUserDoneDirPermissions( + Configuration conf) { + String userDoneDirPermissions = conf.get( + JHAdminConfig.MR_HISTORY_INTERMEDIATE_USER_DONE_DIR_PERMISSIONS); + if (userDoneDirPermissions == null) { + return new FsPermission( + JHAdminConfig.DEFAULT_MR_HISTORY_INTERMEDIATE_USER_DONE_DIR_PERMISSIONS); + } + FsPermission permission = new FsPermission(userDoneDirPermissions); + if (permission.getUserAction() != FsAction.ALL || + permission.getGroupAction() != FsAction.ALL) { + permission = new FsPermission(FsAction.ALL, FsAction.ALL, + permission.getOtherAction(), permission.getStickyBit()); + LOG.warn("Unsupported permission configured in " + + JHAdminConfig.MR_HISTORY_INTERMEDIATE_USER_DONE_DIR_PERMISSIONS + + ", the user and the group permission must be 7 (rwx). " + + "The permission was set to " + permission.toString()); + } + return permission; + } /** * Gets the configured directory prefix for Done history files. http://git-wip-us.apache.org/repos/asf/hadoop/blob/5837a940/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/jobhistory/TestJobHistoryUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/jobhistory/TestJobHistoryUtils.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/jobhistory/TestJobHistoryUtils.java index 6878cce..cee401e 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/jobhistory/TestJobHistoryUtils.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/jobhistory/TestJobHistoryUtils.java @@ -24,6 +24,10 @@ import java.io.IOException; import java.util.Calendar; import java.util.Collections; import java.util.List; +import java.util.Map; + +import com.google.common.collect.ImmutableMap; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileContext; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.Path; @@ -31,6 +35,9 @@ import org.apache.hadoop.fs.permission.FsPermission; import org.junit.Assert; import org.junit.Test; +import static org.apache.hadoop.mapreduce.v2.jobhistory.JobHistoryUtils.getConfiguredHistoryIntermediateUserDoneDirPermissions; + + public class TestJobHistoryUtils { final static String TEST_DIR = new File(System.getProperty("test.build.data")) @@ -140,4 +147,21 @@ public class TestJobHistoryUtils { fc.mkdir(path, FsPermission.getDirDefault(), true); return path; } + + @Test + public void testGetConfiguredHistoryIntermediateUserDoneDirPermissions() { + Configuration conf = new Configuration(); + Map<String, FsPermission> parameters = ImmutableMap.of( + "775", new FsPermission(0775), + "123", new FsPermission(0773), + "-rwx", new FsPermission(0770) , + "+rwx", new FsPermission(0777) + ); + for (Map.Entry<String, FsPermission> entry : parameters.entrySet()) { + conf.set(JHAdminConfig.MR_HISTORY_INTERMEDIATE_USER_DONE_DIR_PERMISSIONS, + entry.getKey()); + Assert.assertEquals(entry.getValue(), + getConfiguredHistoryIntermediateUserDoneDirPermissions(conf)); + } + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/5837a940/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml index 72f509c..d0e5a2d 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml @@ -1731,6 +1731,15 @@ </property> <property> + <name>mapreduce.jobhistory.intermediate-user-done-dir.permissions</name> + <value>770</value> + <description>The permissions of the user directories in + ${mapreduce.jobhistory.intermediate-done-dir}. The user and the group + permission must be 7, this is enforced. + </description> +</property> + +<property> <name>mapreduce.jobhistory.done-dir</name> <value>${yarn.app.mapreduce.am.staging-dir}/history/done</value> <description></description> --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
