svn commit: r734870 - in /hadoop/core/trunk: CHANGES.txt src/mapred/org/apache/hadoop/mapred/TaskTracker.java

[acmurthy]

Author: acmurthy
Date: Thu Jan 15 16:30:26 2009
New Revision: 734870

URL: http://svn.apache.org/viewvc?rev=734870&view=rev
Log:
HADOOP-4906. Fix TaskTracker OOM by keeping a shallow copy of JobConf in 
TaskTracker.TaskInProgress. Contributed by Sharad Agarwal.

Modified:
    hadoop/core/trunk/CHANGES.txt
    hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/TaskTracker.java

Modified: hadoop/core/trunk/CHANGES.txt
URL: 
http://svn.apache.org/viewvc/hadoop/core/trunk/CHANGES.txt?rev=734870&r1=734869&r2=734870&view=diff
==============================================================================
--- hadoop/core/trunk/CHANGES.txt (original)
+++ hadoop/core/trunk/CHANGES.txt Thu Jan 15 16:30:26 2009
@@ -632,6 +632,9 @@
     HADOOP-5008. TestReplication#testPendingReplicationRetry leaves an opened
     fd unclosed. (hairong)
 
+    HADOOP-4906. Fix TaskTracker OOM by keeping a shallow copy of JobConf in
+    TaskTracker.TaskInProgress. (Sharad Agarwal via acmurthy) 
+
 Release 0.19.0 - 2008-11-18
 
   INCOMPATIBLE CHANGES

Modified: hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/TaskTracker.java
URL: 
http://svn.apache.org/viewvc/hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/TaskTracker.java?rev=734870&r1=734869&r2=734870&view=diff
==============================================================================
--- hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/TaskTracker.java 
(original)
+++ hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/TaskTracker.java Thu 
Jan 15 16:30:26 2009
@@ -372,15 +372,13 @@
       }, "taskCleanup");
     
   private RunningJob addTaskToJob(JobID jobId, 
-                                  Path localJobFile,
                                   TaskInProgress tip) {
     synchronized (runningJobs) {
       RunningJob rJob = null;
       if (!runningJobs.containsKey(jobId)) {
-        rJob = new RunningJob(jobId, localJobFile);
+        rJob = new RunningJob(jobId);
         rJob.localized = false;
         rJob.tasks = new HashSet<TaskInProgress>();
-        rJob.jobFile = localJobFile;
         runningJobs.put(jobId, rJob);
       } else {
         rJob = runningJobs.get(jobId);
@@ -762,7 +760,7 @@
                                     + Path.SEPARATOR + jobId 
                                     + Path.SEPARATOR + "job.xml"),
                                     jobFileSize, fConf);
-    RunningJob rjob = addTaskToJob(jobId, localJobFile, tip);
+    RunningJob rjob = addTaskToJob(jobId, tip);
     synchronized (rjob) {
       if (!rjob.localized) {
   
@@ -830,9 +828,10 @@
         rjob.keepJobFiles = ((localJobConf.getKeepTaskFilesPattern() != null) 
||
                              localJobConf.getKeepFailedTaskFiles());
         rjob.localized = true;
+        rjob.jobConf = localJobConf;
       }
     }
-    launchTaskForJob(tip, new JobConf(rjob.jobFile)); 
+    launchTaskForJob(tip, new JobConf(rjob.jobConf)); 
   }
 
   private void launchTaskForJob(TaskInProgress tip, JobConf jobConf) throws 
IOException{
@@ -2605,24 +2604,19 @@
    */
   static class RunningJob{
     private JobID jobid; 
-    private Path jobFile;
+    private JobConf jobConf;
     // keep this for later use
     volatile Set<TaskInProgress> tasks;
     boolean localized;
     boolean keepJobFiles;
     FetchStatus f;
-    RunningJob(JobID jobid, Path jobFile) {
+    RunningJob(JobID jobid) {
       this.jobid = jobid;
       localized = false;
       tasks = new HashSet<TaskInProgress>();
-      this.jobFile = jobFile;
       keepJobFiles = false;
     }
       
-    Path getJobFile() {
-      return jobFile;
-    }
-      
     JobID getJobID() {
       return jobid;
     }