Author: omalley
Date: Fri Mar 4 03:25:52 2011
New Revision: 1076957
URL: http://svn.apache.org/viewvc?rev=1076957&view=rev
Log:
commit 95b712ea53d706cee2450155f670bef9848ec9f9
Author: Lee Tucker <[email protected]>
Date: Thu Jul 30 17:40:44 2009 -0700
Applying patch 2871577.mr693.patch
Modified:
hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/JobHistory.java
hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/JobTracker.java
hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/mapred/TestJobTrackerRestart.java
Modified:
hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/JobHistory.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/JobHistory.java?rev=1076957&r1=1076956&r2=1076957&view=diff
==============================================================================
---
hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/JobHistory.java
(original)
+++
hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/JobHistory.java
Fri Mar 4 03:25:52 2011
@@ -104,6 +104,15 @@ public class JobHistory {
private static JobConf jtConf;
private static Path DONE = null; // folder for completed jobs
/**
+ * A filter for conf files
+ */
+ private static final PathFilter CONF_FILTER = new PathFilter() {
+ public boolean accept(Path path) {
+ return path.getName().endsWith("_conf.xml");
+ }
+ };
+
+ /**
* A class that manages all the files related to a job. For now
* - writers : list of open files
* - job history filename
@@ -938,6 +947,19 @@ public class JobHistory {
}
/**
+ * Delete job conf from the history folder.
+ */
+ static void deleteConfFiles() throws IOException {
+ LOG.info("Cleaning up config files from the job history folder");
+ FileSystem fs = new Path(LOG_DIR).getFileSystem(jtConf);
+ FileStatus[] status = fs.listStatus(new Path(LOG_DIR), CONF_FILTER);
+ for (FileStatus s : status) {
+ LOG.info("Deleting conf file " + s.getPath());
+ fs.delete(s.getPath(), false);
+ }
+ }
+
+ /**
* Move the completed job into the completed folder.
* This assumes that the jobhistory file is closed and all operations on
the
* jobhistory file is complete.
Modified:
hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/JobTracker.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/JobTracker.java?rev=1076957&r1=1076956&r2=1076957&view=diff
==============================================================================
---
hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/JobTracker.java
(original)
+++
hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/JobTracker.java
Fri Mar 4 03:25:52 2011
@@ -1430,6 +1430,14 @@ public class JobTracker implements MRCon
Iterator<JobID> idIter = jobsToRecover.iterator();
JobInProgress job = null;
File jobIdFile = null;
+
+ // 0. Cleanup
+ try {
+ JobHistory.JobInfo.deleteConfFiles();
+ } catch (IOException ioe) {
+ LOG.info("Error in cleaning up job history folder", ioe);
+ }
+
while (idIter.hasNext()) {
JobID id = idIter.next();
LOG.info("Trying to recover details of job " + id);
Modified:
hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/mapred/TestJobTrackerRestart.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/mapred/TestJobTrackerRestart.java?rev=1076957&r1=1076956&r2=1076957&view=diff
==============================================================================
---
hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/mapred/TestJobTrackerRestart.java
(original)
+++
hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/mapred/TestJobTrackerRestart.java
Fri Mar 4 03:25:52 2011
@@ -487,6 +487,11 @@ public class TestJobTrackerRestart exten
String history =
JobHistory.JobInfo.getJobHistoryFileName(jip.getJobConf(), id);
Path historyPath = JobHistory.JobInfo.getJobHistoryLogLocation(history);
+ // get the conf file name
+ String parts[] = history.split("_");
+ // jobtracker-hostname_jobtracker-identifier_conf.xml
+ String jobUniqueString = parts[0] + "_" + parts[1] + "_" + id;
+ Path confPath = new Path(historyPath.getParent(), jobUniqueString +
"_conf.xml");
// make sure that setup is launched
while (jip.runningMaps() == 0) {
@@ -521,6 +526,10 @@ public class TestJobTrackerRestart exten
job1.waitForCompletion();
job2.waitForCompletion();
+
+ // check if the old files are deleted
+ assertFalse("Old jobhistory file is not deleted",
historyFS.exists(historyPath));
+ assertFalse("Old jobconf file is not deleted", historyFS.exists(confPath));
}
public void testJobTrackerRestart() throws IOException {
