Author: sharad
Date: Fri Jun 26 11:34:58 2009
New Revision: 788662
URL: http://svn.apache.org/viewvc?rev=788662&view=rev
Log:
MAPREDUCE-130. Delete the jobconf copy from the log directory of the JobTracker
when the job is retired. Contributed by Amar Kamat.
Modified:
hadoop/common/branches/branch-0.20/CHANGES.txt
hadoop/common/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/JobHistory.java
hadoop/common/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/JobTracker.java
hadoop/common/branches/branch-0.20/src/test/org/apache/hadoop/mapred/TestJobHistory.java
Modified: hadoop/common/branches/branch-0.20/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20/CHANGES.txt?rev=788662&r1=788661&r2=788662&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20/CHANGES.txt (original)
+++ hadoop/common/branches/branch-0.20/CHANGES.txt Fri Jun 26 11:34:58 2009
@@ -147,6 +147,9 @@
MAPREDUCE-2. Fixes a bug in KeyFieldBasedPartitioner in handling empty
keys. (Amar Kamat via sharad)
+ MAPREDUCE-130. Delete the jobconf copy from the log directory of the
+ JobTracker when the job is retired. (Amar Kamat via sharad)
+
Release 0.20.0 - 2009-04-15
INCOMPATIBLE CHANGES
Modified:
hadoop/common/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/JobHistory.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/JobHistory.java?rev=788662&r1=788661&r2=788662&view=diff
==============================================================================
---
hadoop/common/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/JobHistory.java
(original)
+++
hadoop/common/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/JobHistory.java
Fri Jun 26 11:34:58 2009
@@ -833,6 +833,19 @@
}
/**
+ * Deletes job data from the local disk.
+ * For now just deletes the localized copy of job conf
+ */
+ static void cleanupJob(JobID id) {
+ String localJobFilePath = JobInfo.getLocalJobFilePath(id);
+ File f = new File (localJobFilePath);
+ LOG.info("Deleting localized job conf at " + f);
+ if (!f.delete()) {
+ LOG.debug("Failed to delete file " + f);
+ }
+ }
+
+ /**
* Log job submitted event to history. Creates a new file in history
* for the job. if history file creation fails, it disables history
* for all other events.
Modified:
hadoop/common/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/JobTracker.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/JobTracker.java?rev=788662&r1=788661&r2=788662&view=diff
==============================================================================
---
hadoop/common/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/JobTracker.java
(original)
+++
hadoop/common/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/JobTracker.java
Fri Jun 26 11:34:58 2009
@@ -426,6 +426,9 @@
LOG.info("Retired job with id: '" +
job.getProfile().getJobID() + "' of user '" +
jobUser + "'");
+
+ // clean up job files from the local disk
+ JobHistory.JobInfo.cleanupJob(job.getProfile().getJobID());
}
}
}
Modified:
hadoop/common/branches/branch-0.20/src/test/org/apache/hadoop/mapred/TestJobHistory.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20/src/test/org/apache/hadoop/mapred/TestJobHistory.java?rev=788662&r1=788661&r2=788662&view=diff
==============================================================================
---
hadoop/common/branches/branch-0.20/src/test/org/apache/hadoop/mapred/TestJobHistory.java
(original)
+++
hadoop/common/branches/branch-0.20/src/test/org/apache/hadoop/mapred/TestJobHistory.java
Fri Jun 26 11:34:58 2009
@@ -778,10 +778,14 @@
public void testJobHistoryFile() throws IOException {
MiniMRCluster mr = null;
try {
- mr = new MiniMRCluster(2, "file:///", 3);
+ JobConf conf = new JobConf();
+ // keep for less time
+ conf.setLong("mapred.jobtracker.retirejob.check", 1000);
+ conf.setLong("mapred.jobtracker.retirejob.interval", 1000);
+ mr = new MiniMRCluster(2, "file:///", 3, null, null, conf);
// run the TCs
- JobConf conf = mr.createJobConf();
+ conf = mr.createJobConf();
FileSystem fs = FileSystem.get(conf);
// clean up
@@ -802,6 +806,15 @@
validateJobHistoryFileFormat(job.getID(), conf, "SUCCESS", false);
validateJobHistoryFileContent(mr, job, conf);
+ // get the job conf filename
+ String name = JobHistory.JobInfo.getLocalJobFilePath(job.getID());
+ File file = new File(name);
+
+ // check if the file get deleted
+ while (file.exists()) {
+ LOG.info("Waiting for " + file + " to be deleted");
+ UtilsForTests.waitFor(100);
+ }
} finally {
if (mr != null) {
cleanupLocalFiles(mr);
