Author: ddas
Date: Fri Dec 5 04:28:07 2008
New Revision: 723722
URL: http://svn.apache.org/viewvc?rev=723722&view=rev
Log:
HADOOP-4654. Removes temporary output directory for failed and killed tasks in
the JobTracker's task commit thread. Contributed by Amareshwari Sriramadasu.
Modified:
hadoop/core/branches/branch-0.18/CHANGES.txt
hadoop/core/branches/branch-0.18/src/mapred/org/apache/hadoop/mapred/JobInProgress.java
hadoop/core/branches/branch-0.18/src/mapred/org/apache/hadoop/mapred/JobTracker.java
hadoop/core/branches/branch-0.18/src/mapred/org/apache/hadoop/mapred/Task.java
Modified: hadoop/core/branches/branch-0.18/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.18/CHANGES.txt?rev=723722&r1=723721&r2=723722&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.18/CHANGES.txt (original)
+++ hadoop/core/branches/branch-0.18/CHANGES.txt Fri Dec 5 04:28:07 2008
@@ -71,6 +71,10 @@
HADOOP-4746. Job output directory should be normalized. (hairong)
+ HADOOP-4654. Removes temporary output directory for failed and killed
+ tasks in the JobTracker's task commit thread.
+ (Amareshwari Sriramadasu via ddas)
+
Release 0.18.2 - 2008-11-03
BUG FIXES
Modified:
hadoop/core/branches/branch-0.18/src/mapred/org/apache/hadoop/mapred/JobInProgress.java
URL:
http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.18/src/mapred/org/apache/hadoop/mapred/JobInProgress.java?rev=723722&r1=723721&r2=723722&view=diff
==============================================================================
---
hadoop/core/branches/branch-0.18/src/mapred/org/apache/hadoop/mapred/JobInProgress.java
(original)
+++
hadoop/core/branches/branch-0.18/src/mapred/org/apache/hadoop/mapred/JobInProgress.java
Fri Dec 5 04:28:07 2008
@@ -524,6 +524,10 @@
return;
} else if (state == TaskStatus.State.FAILED ||
state == TaskStatus.State.KILLED) {
+ // To remove the temporary output of failed/killed tasks
+ JobWithTaskContext j = new JobWithTaskContext(this, tip,
+ status.getTaskID(), metrics);
+ jobtracker.addToCommitQueue(j);
// Get the event number for the (possibly) previously successful
// task. If there exists one, then set that status to OBSOLETE
int eventNumber;
Modified:
hadoop/core/branches/branch-0.18/src/mapred/org/apache/hadoop/mapred/JobTracker.java
URL:
http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.18/src/mapred/org/apache/hadoop/mapred/JobTracker.java?rev=723722&r1=723721&r2=723722&view=diff
==============================================================================
---
hadoop/core/branches/branch-0.18/src/mapred/org/apache/hadoop/mapred/JobTracker.java
(original)
+++
hadoop/core/branches/branch-0.18/src/mapred/org/apache/hadoop/mapred/JobTracker.java
Fri Dec 5 04:28:07 2008
@@ -2318,6 +2318,15 @@
isTipComplete[index] = true;
}
}
+ } else if (states[index] == TaskStatus.State.FAILED ||
+ states[index] == TaskStatus.State.KILLED) {
+ try {
+ tasks[index].removeTaskOutput();
+ } catch (IOException e) {
+ LOG.info("Failed to remove temporary directory of "
+ + status[index].getTaskID() + " with "
+ + StringUtils.stringifyException(e));
+ }
}
} catch (IOException ioe) {
// Oops! Failed to copy the task's output to its final place;
Modified:
hadoop/core/branches/branch-0.18/src/mapred/org/apache/hadoop/mapred/Task.java
URL:
http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.18/src/mapred/org/apache/hadoop/mapred/Task.java?rev=723722&r1=723721&r2=723722&view=diff
==============================================================================
---
hadoop/core/branches/branch-0.18/src/mapred/org/apache/hadoop/mapred/Task.java
(original)
+++
hadoop/core/branches/branch-0.18/src/mapred/org/apache/hadoop/mapred/Task.java
Fri Dec 5 04:28:07 2008
@@ -571,6 +571,14 @@
}
}
+ void removeTaskOutput() throws IOException {
+ if (taskOutputPath != null) {
+ FileSystem fs = taskOutputPath.getFileSystem(conf);
+ // Delete the temporary task-specific output directory
+ fs.delete(taskOutputPath, true);
+ }
+ }
+
private Path getFinalPath(Path jobOutputDir, Path taskOutput) {
URI relativePath = taskOutputPath.toUri().relativize(taskOutput.toUri());
if (relativePath.getPath().length() > 0) {
