Author: acmurthy
Date: Thu Jan 15 16:34:41 2009
New Revision: 734873
URL: http://svn.apache.org/viewvc?rev=734873&view=rev
Log:
Merge -r 734869:734870 from trunk to branch-0.19 to fix HADOOP-4906.
Modified:
hadoop/core/branches/branch-0.19/ (props changed)
hadoop/core/branches/branch-0.19/CHANGES.txt (contents, props changed)
hadoop/core/branches/branch-0.19/src/mapred/org/apache/hadoop/mapred/TaskTracker.java
Propchange: hadoop/core/branches/branch-0.19/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Jan 15 16:34:41 2009
@@ -1 +1 @@
-/hadoop/core/trunk:697306,698176,699056,699098,699415,699424,699444,699490,699517,700163,700628,700923,701273,701398,703923,704203,704261,704701,704703,704707,704712,704732,704748,704989,705391,705420,705430,705762,706350,706707,706719,706796,706802,707258,707262,708623,708641,708710,709040,709303,712881,713888,720602,723013,723460,723831,723918,724883,727117,727212,727217,727228,727869,732572,732777,733887
+/hadoop/core/trunk:697306,698176,699056,699098,699415,699424,699444,699490,699517,700163,700628,700923,701273,701398,703923,704203,704261,704701,704703,704707,704712,704732,704748,704989,705391,705420,705430,705762,706350,706707,706719,706796,706802,707258,707262,708623,708641,708710,709040,709303,712881,713888,720602,723013,723460,723831,723918,724883,727117,727212,727217,727228,727869,732572,732777,733887,734870
Modified: hadoop/core/branches/branch-0.19/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.19/CHANGES.txt?rev=734873&r1=734872&r2=734873&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.19/CHANGES.txt (original)
+++ hadoop/core/branches/branch-0.19/CHANGES.txt Thu Jan 15 16:34:41 2009
@@ -49,6 +49,9 @@
HADOOP-5008. TestReplication#testPendingReplicationRetry leaves an opened
fd unclosed. (hairong)
+ HADOOP-4906. Fix TaskTracker OOM by keeping a shallow copy of JobConf in
+ TaskTracker.TaskInProgress. (Sharad Agarwal via acmurthy)
+
Release 0.19.0 - 2008-11-18
INCOMPATIBLE CHANGES
Propchange: hadoop/core/branches/branch-0.19/CHANGES.txt
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Jan 15 16:34:41 2009
@@ -1,2 +1,2 @@
/hadoop/core/branches/branch-0.18/CHANGES.txt:727226
-/hadoop/core/trunk/CHANGES.txt:697306,698176,699056,699098,699415,699424,699444,699490,699517,700163,700628,700923,701273,701398,703923,704203,704261,704701,704703,704707,704712,704732,704748,704989,705391,705420,705430,705762,706350,706707,706719,706796,706802,707258,707262,708623,708641,708710,708723,709040,709303,711717,712881,713888,720602,723013,723460,723831,723918,724883,727117,727212,727217,727228,727869,732572,732777,733887
+/hadoop/core/trunk/CHANGES.txt:697306,698176,699056,699098,699415,699424,699444,699490,699517,700163,700628,700923,701273,701398,703923,704203,704261,704701,704703,704707,704712,704732,704748,704989,705391,705420,705430,705762,706350,706707,706719,706796,706802,707258,707262,708623,708641,708710,708723,709040,709303,711717,712881,713888,720602,723013,723460,723831,723918,724883,727117,727212,727217,727228,727869,732572,732777,733887,734870
Modified:
hadoop/core/branches/branch-0.19/src/mapred/org/apache/hadoop/mapred/TaskTracker.java
URL:
http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.19/src/mapred/org/apache/hadoop/mapred/TaskTracker.java?rev=734873&r1=734872&r2=734873&view=diff
==============================================================================
---
hadoop/core/branches/branch-0.19/src/mapred/org/apache/hadoop/mapred/TaskTracker.java
(original)
+++
hadoop/core/branches/branch-0.19/src/mapred/org/apache/hadoop/mapred/TaskTracker.java
Thu Jan 15 16:34:41 2009
@@ -318,15 +318,13 @@
}, "taskCleanup");
private RunningJob addTaskToJob(JobID jobId,
- Path localJobFile,
TaskInProgress tip) {
synchronized (runningJobs) {
RunningJob rJob = null;
if (!runningJobs.containsKey(jobId)) {
- rJob = new RunningJob(jobId, localJobFile);
+ rJob = new RunningJob(jobId);
rJob.localized = false;
rJob.tasks = new HashSet<TaskInProgress>();
- rJob.jobFile = localJobFile;
runningJobs.put(jobId, rJob);
} else {
rJob = runningJobs.get(jobId);
@@ -705,7 +703,7 @@
+ Path.SEPARATOR + jobId
+ Path.SEPARATOR + "job.xml"),
jobFileSize, fConf);
- RunningJob rjob = addTaskToJob(jobId, localJobFile, tip);
+ RunningJob rjob = addTaskToJob(jobId, tip);
synchronized (rjob) {
if (!rjob.localized) {
@@ -773,9 +771,10 @@
rjob.keepJobFiles = ((localJobConf.getKeepTaskFilesPattern() != null)
||
localJobConf.getKeepFailedTaskFiles());
rjob.localized = true;
+ rjob.jobConf = localJobConf;
}
}
- launchTaskForJob(tip, new JobConf(rjob.jobFile));
+ launchTaskForJob(tip, new JobConf(rjob.jobConf));
}
private void launchTaskForJob(TaskInProgress tip, JobConf jobConf) throws
IOException{
@@ -2531,24 +2530,19 @@
*/
static class RunningJob{
private JobID jobid;
- private Path jobFile;
+ private JobConf jobConf;
// keep this for later use
volatile Set<TaskInProgress> tasks;
boolean localized;
boolean keepJobFiles;
FetchStatus f;
- RunningJob(JobID jobid, Path jobFile) {
+ RunningJob(JobID jobid) {
this.jobid = jobid;
localized = false;
tasks = new HashSet<TaskInProgress>();
- this.jobFile = jobFile;
keepJobFiles = false;
}
- Path getJobFile() {
- return jobFile;
- }
-
JobID getJobID() {
return jobid;
}
