MAPREDUCE-6044. Fully qualified intermediate done dir path breaks per-user dir creation on Windows. Contributed by Zhijie Shen.
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1619863 13f79535-47bb-0310-9956-ffa450edef68 Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/524a63e5 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/524a63e5 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/524a63e5 Branch: refs/heads/HDFS-6584 Commit: 524a63e59ecfda99c66842462e1bac5d610e7997 Parents: 0097b15 Author: Zhijie Shen <[email protected]> Authored: Fri Aug 22 17:06:23 2014 +0000 Committer: Zhijie Shen <[email protected]> Committed: Fri Aug 22 17:06:23 2014 +0000 ---------------------------------------------------------------------- hadoop-mapreduce-project/CHANGES.txt | 3 ++ .../jobhistory/TestJobHistoryEventHandler.java | 30 +++++++++++++++++++- .../v2/jobhistory/JobHistoryUtils.java | 4 +-- 3 files changed, 34 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/524a63e5/hadoop-mapreduce-project/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index dfef8e5..ddf21ed 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -255,6 +255,9 @@ Release 2.6.0 - UNRELEASED MAPREDUCE-6012. DBInputSplit creates invalid ranges on Oracle. (Wei Yan via kasha) + MAPREDUCE-6044. Fully qualified intermediate done dir path breaks per-user dir + creation on Windows. (zjshen) + Release 2.5.0 - 2014-08-11 INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/524a63e5/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java index d8a0cc7..7539e73 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java @@ -25,7 +25,6 @@ import static org.mockito.Mockito.spy; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; -import static org.mockito.Mockito.never; import java.io.File; import java.io.FileOutputStream; @@ -53,6 +52,8 @@ import org.apache.hadoop.mapreduce.v2.api.records.JobId; import org.apache.hadoop.mapreduce.v2.app.AppContext; import org.apache.hadoop.mapreduce.v2.app.job.Job; import org.apache.hadoop.mapreduce.v2.app.job.JobStateInternal; +import org.apache.hadoop.mapreduce.v2.jobhistory.JHAdminConfig; +import org.apache.hadoop.mapreduce.v2.jobhistory.JobHistoryUtils; import org.apache.hadoop.mapreduce.v2.util.MRBuilderUtils; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; @@ -399,6 +400,33 @@ public class TestJobHistoryEventHandler { } } + @Test + public void testGetHistoryIntermediateDoneDirForUser() throws IOException { + // Test relative path + Configuration conf = new Configuration(); + conf.set(JHAdminConfig.MR_HISTORY_INTERMEDIATE_DONE_DIR, + "/mapred/history/done_intermediate"); + conf.set(MRJobConfig.USER_NAME, System.getProperty("user.name")); + String pathStr = JobHistoryUtils.getHistoryIntermediateDoneDirForUser(conf); + Assert.assertEquals("/mapred/history/done_intermediate/" + + System.getProperty("user.name"), pathStr); + + // Test fully qualified path + // Create default configuration pointing to the minicluster + conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, + dfsCluster.getURI().toString()); + FileOutputStream os = new FileOutputStream(coreSitePath); + conf.writeXml(os); + os.close(); + // Simulate execution under a non-default namenode + conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, + "file:///"); + pathStr = JobHistoryUtils.getHistoryIntermediateDoneDirForUser(conf); + Assert.assertEquals(dfsCluster.getURI().toString() + + "/mapred/history/done_intermediate/" + System.getProperty("user.name"), + pathStr); + } + private void queueEvent(JHEvenHandlerForTest jheh, JobHistoryEvent event) { jheh.handle(event); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/524a63e5/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JobHistoryUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JobHistoryUtils.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JobHistoryUtils.java index 167ee20..e279c03 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JobHistoryUtils.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JobHistoryUtils.java @@ -292,8 +292,8 @@ public class JobHistoryUtils { * @return the intermediate done directory for jobhistory files. */ public static String getHistoryIntermediateDoneDirForUser(Configuration conf) throws IOException { - return getConfiguredHistoryIntermediateDoneDirPrefix(conf) + File.separator - + UserGroupInformation.getCurrentUser().getShortUserName(); + return new Path(getConfiguredHistoryIntermediateDoneDirPrefix(conf), + UserGroupInformation.getCurrentUser().getShortUserName()).toString(); } public static boolean shouldCreateNonUserDirectory(Configuration conf) {
