Repository: hadoop Updated Branches: refs/heads/trunk 48ca23def -> 6785661e5
MAPREDUCE-6350. JobHistory doesn't support fully-functional search. Contributed by Siqi Li. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6785661e Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6785661e Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6785661e Branch: refs/heads/trunk Commit: 6785661e554114a4613b5fe7dabec9bfa80c41d9 Parents: 48ca23d Author: Devaraj K <[email protected]> Authored: Wed Jun 10 12:40:29 2015 +0530 Committer: Devaraj K <[email protected]> Committed: Wed Jun 10 12:40:29 2015 +0530 ---------------------------------------------------------------------- hadoop-mapreduce-project/CHANGES.txt | 3 +++ .../jobhistory/JobHistoryEventHandler.java | 6 ++++- .../v2/jobhistory/FileNameIndexUtils.java | 17 ++++++++----- .../mapreduce/v2/jobhistory/JHAdminConfig.java | 7 ++++++ .../v2/jobhistory/TestFileNameIndexUtils.java | 26 ++++++++++++++++++++ .../src/main/resources/mapred-default.xml | 8 ++++++ 6 files changed, 60 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/6785661e/hadoop-mapreduce-project/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 115fc1b..3c1884d 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -465,6 +465,9 @@ Release 2.8.0 - UNRELEASED MAPREDUCE-6388. Remove deprecation warnings from JobHistoryServer classes (Ray Chiang via ozawa). + MAPREDUCE-6350. JobHistory doesn't support fully-functional search. + (Siqi Li via devaraj) + Release 2.7.1 - UNRELEASED INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/6785661e/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java index bf32888..35556a6 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java @@ -55,6 +55,7 @@ import org.apache.hadoop.mapreduce.v2.app.AppContext; import org.apache.hadoop.mapreduce.v2.app.job.Job; import org.apache.hadoop.mapreduce.v2.app.job.JobStateInternal; import org.apache.hadoop.mapreduce.v2.jobhistory.FileNameIndexUtils; +import org.apache.hadoop.mapreduce.v2.jobhistory.JHAdminConfig; import org.apache.hadoop.mapreduce.v2.jobhistory.JobHistoryUtils; import org.apache.hadoop.mapreduce.v2.jobhistory.JobIndexInfo; import org.apache.hadoop.security.UserGroupInformation; @@ -1107,9 +1108,12 @@ public class JobHistoryEventHandler extends AbstractService if (mi.getHistoryFile() != null) { Path historyFile = mi.getHistoryFile(); Path qualifiedLogFile = stagingDirFS.makeQualified(historyFile); + int jobNameLimit = + getConfig().getInt(JHAdminConfig.MR_HS_JOBNAME_LIMIT, + JHAdminConfig.DEFAULT_MR_HS_JOBNAME_LIMIT); String doneJobHistoryFileName = getTempFileName(FileNameIndexUtils.getDoneFileName(mi - .getJobIndexInfo())); + .getJobIndexInfo(), jobNameLimit)); qualifiedDoneFile = doneDirFS.makeQualified(new Path(doneDirPrefixPath, doneJobHistoryFileName)); http://git-wip-us.apache.org/repos/asf/hadoop/blob/6785661e/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/FileNameIndexUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/FileNameIndexUtils.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/FileNameIndexUtils.java index 741da11..eb0c54c 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/FileNameIndexUtils.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/FileNameIndexUtils.java @@ -32,8 +32,6 @@ import org.apache.hadoop.mapreduce.v2.api.records.JobId; public class FileNameIndexUtils { - static final int JOB_NAME_TRIM_LENGTH = 50; - // Sanitize job history file for predictable parsing static final String DELIMITER = "-"; static final String DELIMITER_ESCAPE = "%2D"; @@ -60,6 +58,12 @@ public class FileNameIndexUtils { * @return the done job history filename. */ public static String getDoneFileName(JobIndexInfo indexInfo) throws IOException { + return getDoneFileName(indexInfo, + JHAdminConfig.DEFAULT_MR_HS_JOBNAME_LIMIT); + } + + public static String getDoneFileName(JobIndexInfo indexInfo, + int jobNameLimit) throws IOException { StringBuilder sb = new StringBuilder(); //JobId sb.append(escapeDelimiters(TypeConverter.fromYarn(indexInfo.getJobId()).toString())); @@ -74,7 +78,8 @@ public class FileNameIndexUtils { sb.append(DELIMITER); //JobName - sb.append(escapeDelimiters(trimJobName(getJobName(indexInfo)))); + sb.append(escapeDelimiters(trimJobName( + getJobName(indexInfo), jobNameLimit))); sb.append(DELIMITER); //FinishTime @@ -286,9 +291,9 @@ public class FileNameIndexUtils { /** * Trims the job-name if required */ - private static String trimJobName(String jobName) { - if (jobName.length() > JOB_NAME_TRIM_LENGTH) { - jobName = jobName.substring(0, JOB_NAME_TRIM_LENGTH); + private static String trimJobName(String jobName, int jobNameLimit) { + if (jobName.length() > jobNameLimit) { + jobName = jobName.substring(0, jobNameLimit); } return jobName; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/6785661e/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java index f7cba9f..a97c2ca 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java @@ -214,4 +214,11 @@ public class JHAdminConfig { */ public static boolean DEFAULT_MR_HISTORY_MINICLUSTER_FIXED_PORTS = false; + /** + * Number of characters allowed for job name in Job History Server web page. + */ + public static final String MR_HS_JOBNAME_LIMIT = MR_HISTORY_PREFIX + + "jobname.limit"; + public static final int DEFAULT_MR_HS_JOBNAME_LIMIT = 50; + } http://git-wip-us.apache.org/repos/asf/hadoop/blob/6785661e/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/jobhistory/TestFileNameIndexUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/jobhistory/TestFileNameIndexUtils.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/jobhistory/TestFileNameIndexUtils.java index a0d5fce..3d722e0 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/jobhistory/TestFileNameIndexUtils.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/jobhistory/TestFileNameIndexUtils.java @@ -148,6 +148,32 @@ public class TestFileNameIndexUtils { } @Test + public void testTrimJobName() throws IOException { + int jobNameTrimLength = 5; + JobIndexInfo info = new JobIndexInfo(); + JobID oldJobId = JobID.forName(JOB_ID); + JobId jobId = TypeConverter.toYarn(oldJobId); + info.setJobId(jobId); + info.setSubmitTime(Long.parseLong(SUBMIT_TIME)); + info.setUser(USER_NAME); + info.setJobName(JOB_NAME); + info.setFinishTime(Long.parseLong(FINISH_TIME)); + info.setNumMaps(Integer.parseInt(NUM_MAPS)); + info.setNumReduces(Integer.parseInt(NUM_REDUCES)); + info.setJobStatus(JOB_STATUS); + info.setQueueName(QUEUE_NAME); + info.setJobStartTime(Long.parseLong(JOB_START_TIME)); + + String jobHistoryFile = + FileNameIndexUtils.getDoneFileName(info, jobNameTrimLength); + JobIndexInfo parsedInfo = FileNameIndexUtils.getIndexInfo(jobHistoryFile); + + Assert.assertEquals("Job name did not get trimmed correctly", + info.getJobName().substring(0, jobNameTrimLength), + parsedInfo.getJobName()); + } + + @Test public void testUserNamePercentDecoding() throws IOException { String jobHistoryFile = String.format(JOB_HISTORY_FILE_FORMATTER, JOB_ID, http://git-wip-us.apache.org/repos/asf/hadoop/blob/6785661e/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml index 4a2dd77..3789429 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml @@ -1706,6 +1706,14 @@ </property> <property> + <name>mapreduce.jobhistory.jobname.limit</name> + <value>50</value> + <description> + Number of characters allowed for job name in Job History Server web page. + </description> +</property> + +<property> <name>mapreduce.job.heap.memory-mb.ratio</name> <value>0.8</value> <description>The ratio of heap-size to container-size. If no -Xmx is
