Author: ddas
Date: Thu May 21 09:00:06 2009
New Revision: 777025
URL: http://svn.apache.org/viewvc?rev=777025&view=rev
Log:
Merge -r 772883:772884 from trunk onto 0.20 branch. Fixes HADOOP-4372.
Modified:
hadoop/core/branches/branch-0.20/ (props changed)
hadoop/core/branches/branch-0.20/CHANGES.txt (contents, props changed)
hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/JobHistory.java
hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/JobInProgress.java
hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/JobTracker.java
Propchange: hadoop/core/branches/branch-0.20/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu May 21 09:00:06 2009
@@ -1,2 +1,2 @@
/hadoop/core/branches/branch-0.19:713112
-/hadoop/core/trunk:727001,727117,727191,727212,727217,727228,727255,727869,728187,729052,729987,732385,732572,732613,732777,732838,732869,733887,734870,734916,736426,738328,738697,740077,740157,741703,741762,743745,743816,743892,744894,745180,746010,746206,746227,746233,746274,746338,746902-746903,746925,746944,746968,746970,747279,747289,747802,748084,748090,748783,749262,749318,749863,750533,752073,752609,752834,752836,752913,752932,753112-753113,753346,754645,754847,754927,755035,755226,755348,755370,755418,755426,755790,755905,755938,755960,755986,755998,756352,757448,757624,757849,758156,758180,759398,759932,760502,760783,761046,761482,761632,762216,762879,763107,763502,764967,765016,765809,765951,771607,771661,772844,772876,772920,773889,776638
+/hadoop/core/trunk:727001,727117,727191,727212,727217,727228,727255,727869,728187,729052,729987,732385,732572,732613,732777,732838,732869,733887,734870,734916,736426,738328,738697,740077,740157,741703,741762,743745,743816,743892,744894,745180,746010,746206,746227,746233,746274,746338,746902-746903,746925,746944,746968,746970,747279,747289,747802,748084,748090,748783,749262,749318,749863,750533,752073,752609,752834,752836,752913,752932,753112-753113,753346,754645,754847,754927,755035,755226,755348,755370,755418,755426,755790,755905,755938,755960,755986,755998,756352,757448,757624,757849,758156,758180,759398,759932,760502,760783,761046,761482,761632,762216,762879,763107,763502,764967,765016,765809,765951,771607,771661,772844,772876,772884,772920,773889,776638
Modified: hadoop/core/branches/branch-0.20/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.20/CHANGES.txt?rev=777025&r1=777024&r2=777025&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.20/CHANGES.txt (original)
+++ hadoop/core/branches/branch-0.20/CHANGES.txt Thu May 21 09:00:06 2009
@@ -20,6 +20,9 @@
HADOOP-4674. Fix fs help messages for -test, -text, -tail, -stat
and -touchz options. (Ravi Phulari via szetszwo)
+ HADOOP-4372. Improves the way history filenames are obtained and
manipulated.
+ (Amar Kamat via ddas)
+
OPTIMIZATIONS
BUG FIXES
Propchange: hadoop/core/branches/branch-0.20/CHANGES.txt
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu May 21 09:00:06 2009
@@ -1,3 +1,3 @@
/hadoop/core/branches/branch-0.18/CHANGES.txt:727226
/hadoop/core/branches/branch-0.19/CHANGES.txt:713112
-/hadoop/core/trunk/CHANGES.txt:727001,727117,727191,727212,727228,727255,727869,728187,729052,729987,732385,732572,732613,732777,732838,732869,733887,734870,734916,735082,736426,738602,738697,739416,740077,740157,741703,741762,743296,743745,743816,743892,744894,745180,745268,746010,746193,746206,746227,746233,746274,746902-746903,746925,746944,746968,746970,747279,747289,747802,748084,748090,748783,749262,749318,749863,750533,752073,752514,752555,752590,752609,752834,752836,752913,752932,753112-753113,753346,754645,754847,754927,755035,755226,755348,755370,755418,755426,755790,755905,755938,755986,755998,756352,757448,757624,757849,758156,758180,759398,759932,760502,760783,761046,761482,761632,762216,762879,763107,763502,764967,765016,765809,765951,771607,772844,772876,772920,773889,776638
+/hadoop/core/trunk/CHANGES.txt:727001,727117,727191,727212,727228,727255,727869,728187,729052,729987,732385,732572,732613,732777,732838,732869,733887,734870,734916,735082,736426,738602,738697,739416,740077,740157,741703,741762,743296,743745,743816,743892,744894,745180,745268,746010,746193,746206,746227,746233,746274,746902-746903,746925,746944,746968,746970,747279,747289,747802,748084,748090,748783,749262,749318,749863,750533,752073,752514,752555,752590,752609,752834,752836,752913,752932,753112-753113,753346,754645,754847,754927,755035,755226,755348,755370,755418,755426,755790,755905,755938,755986,755998,756352,757448,757624,757849,758156,758180,759398,759932,760502,760783,761046,761482,761632,762216,762879,763107,763502,764967,765016,765809,765951,771607,772844,772876,772884,772920,773889,776638
Modified:
hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/JobHistory.java
URL:
http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/JobHistory.java?rev=777025&r1=777024&r2=777025&view=diff
==============================================================================
---
hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/JobHistory.java
(original)
+++
hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/JobHistory.java
Thu May 21 09:00:06 2009
@@ -666,12 +666,9 @@
};
FileStatus[] statuses = fs.listStatus(new Path(LOG_DIR), filter);
- String filename;
+ String filename = null;
if (statuses.length == 0) {
- filename =
- encodeJobHistoryFileName(getNewJobHistoryFileName(jobConf, id));
- LOG.info("Nothing to recover! Generating a new filename " + filename
- + " for job " + id);
+ LOG.info("Nothing to recover for job " + id);
} else {
// return filename considering that fact the name can be a
// secondary filename like filename.recover
@@ -792,6 +789,9 @@
throws IOException {
String masterLogFileName =
JobHistory.JobInfo.getJobHistoryFileName(conf, id);
+ if (masterLogFileName == null) {
+ return;
+ }
Path masterLogPath =
JobHistory.JobInfo.getJobHistoryLogLocation(masterLogFileName);
String tmpLogFileName = getSecondaryJobHistoryFile(masterLogFileName);
@@ -834,10 +834,19 @@
* @param jobConfPath path to job conf xml file in HDFS.
* @param submitTime time when job tracker received the job
* @throws IOException
+ * @deprecated Use
+ * {...@link #logSubmitted(JobID, JobConf, String, long, boolean)}
instead.
*/
public static void logSubmitted(JobID jobId, JobConf jobConf,
String jobConfPath, long submitTime)
throws IOException {
+ logSubmitted(jobId, jobConf, jobConfPath, submitTime, true);
+ }
+
+ public static void logSubmitted(JobID jobId, JobConf jobConf,
+ String jobConfPath, long submitTime,
+ boolean restarted)
+ throws IOException {
FileSystem fs = null;
String userLogDir = null;
String jobUniqueString = JOBTRACKER_UNIQUE_STRING + jobId;
@@ -850,8 +859,13 @@
String user = getUserName(jobConf);
// get the history filename
- String logFileName =
- getJobHistoryFileName(jobConf, jobId);
+ String logFileName = null;
+ if (restarted) {
+ logFileName = getJobHistoryFileName(jobConf, jobId);
+ } else {
+ logFileName =
+ encodeJobHistoryFileName(getNewJobHistoryFileName(jobConf, jobId));
+ }
// setup the history log file for this job
Path logFile = getJobHistoryLogLocation(logFileName);
@@ -869,8 +883,10 @@
// create output stream for logging in hadoop.job.history.location
fs = new Path(LOG_DIR).getFileSystem(jobConf);
- logFile = recoverJobHistoryFile(jobConf, logFile);
- logFileName = logFile.getName();
+ if (restarted) {
+ logFile = recoverJobHistoryFile(jobConf, logFile);
+ logFileName = logFile.getName();
+ }
int defaultBufferSize =
fs.getConf().getInt("io.file.buffer.size", 4096);
Modified:
hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/JobInProgress.java
URL:
http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/JobInProgress.java?rev=777025&r1=777024&r2=777025&view=diff
==============================================================================
---
hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/JobInProgress.java
(original)
+++
hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/JobInProgress.java
Thu May 21 09:00:06 2009
@@ -373,6 +373,10 @@
return tasksInited.get();
}
+ boolean hasRestarted() {
+ return restartCount > 0;
+ }
+
/**
* Construct the splits, etc. This is invoked from an async
* thread so that split-computation doesn't block anyone.
@@ -392,7 +396,7 @@
// log job info
JobHistory.JobInfo.logSubmitted(getJobID(), conf, jobFile.toString(),
- this.startTime);
+ this.startTime, hasRestarted());
// log the job priority
setPriority(this.priority);
Modified:
hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/JobTracker.java
URL:
http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/JobTracker.java?rev=777025&r1=777024&r2=777025&view=diff
==============================================================================
---
hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/JobTracker.java
(original)
+++
hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/JobTracker.java
Thu May 21 09:00:06 2009
@@ -1234,18 +1234,23 @@
// 3. Get the log file and the file path
String logFileName =
JobHistory.JobInfo.getJobHistoryFileName(job.getJobConf(), id);
- Path jobHistoryFilePath =
- JobHistory.JobInfo.getJobHistoryLogLocation(logFileName);
-
- // 4. Recover the history file. This involved
- // - deleting file.recover if file exists
- // - renaming file.recover to file if file doesnt exist
- // This makes sure that the (master) file exists
- JobHistory.JobInfo.recoverJobHistoryFile(job.getJobConf(),
- jobHistoryFilePath);
+ if (logFileName != null) {
+ Path jobHistoryFilePath =
+ JobHistory.JobInfo.getJobHistoryLogLocation(logFileName);
+
+ // 4. Recover the history file. This involved
+ // - deleting file.recover if file exists
+ // - renaming file.recover to file if file doesnt exist
+ // This makes sure that the (master) file exists
+ JobHistory.JobInfo.recoverJobHistoryFile(job.getJobConf(),
+ jobHistoryFilePath);
- // 5. Cache the history file name as it costs one dfs access
- jobHistoryFilenameMap.put(job.getJobID(), jobHistoryFilePath);
+ // 5. Cache the history file name as it costs one dfs access
+ jobHistoryFilenameMap.put(job.getJobID(), jobHistoryFilePath);
+ } else {
+ LOG.info("No history file found for job " + id);
+ idIter.remove(); // remove from recovery list
+ }
// 6. Sumbit the job to the jobtracker
addJob(id, job);
@@ -2020,10 +2025,12 @@
// start the merge of log files
JobID id = job.getStatus().getJobID();
- try {
- JobHistory.JobInfo.finalizeRecovery(id, job.getJobConf());
- } catch (IOException ioe) {
- LOG.info("Failed to finalize the log file recovery for job " + id, ioe);
+ if (job.hasRestarted()) {
+ try {
+ JobHistory.JobInfo.finalizeRecovery(id, job.getJobConf());
+ } catch (IOException ioe) {
+ LOG.info("Failed to finalize the log file recovery for job " + id,
ioe);
+ }
}
final JobTrackerInstrumentation metrics = getInstrumentation();
