svn commit: r677264 - in /hadoop/core/trunk: ./ src/mapred/org/apache/hadoop/mapred/

[ddas]

Author: ddas
Date: Wed Jul 16 05:44:18 2008
New Revision: 677264

URL: http://svn.apache.org/viewvc?rev=677264&view=rev
Log:
HADOOP-3670. Fixes JobTracker to clear out split bytes when no longer required. 
Contributed by Amareshwari Sriramadasu.

Modified:
    hadoop/core/trunk/CHANGES.txt
    hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JobClient.java
    hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JobInProgress.java
    hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/MapTask.java
    hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/TaskInProgress.java

Modified: hadoop/core/trunk/CHANGES.txt
URL: 
http://svn.apache.org/viewvc/hadoop/core/trunk/CHANGES.txt?rev=677264&r1=677263&r2=677264&view=diff
==============================================================================
--- hadoop/core/trunk/CHANGES.txt (original)
+++ hadoop/core/trunk/CHANGES.txt Wed Jul 16 05:44:18 2008
@@ -843,6 +843,9 @@
     HADOOP-3737. Fix CompressedWritable to call Deflater::end to release
     compressor memory. (Grant Glouser via cdouglas)
 
+    HADOOP-3670. Fixes JobTracker to clear out split bytes when no longer 
+    required. (Amareshwari Sriramadasu via ddas)
+
 Release 0.17.2 - Unreleased
 
   BUG FIXES

Modified: hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JobClient.java
URL: 
http://svn.apache.org/viewvc/hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JobClient.java?rev=677264&r1=677263&r2=677264&view=diff
==============================================================================
--- hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JobClient.java 
(original)
+++ hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JobClient.java Wed 
Jul 16 05:44:18 2008
@@ -757,6 +757,10 @@
     public BytesWritable getBytes() {
       return bytes;
     }
+
+    public void clearBytes() {
+      bytes = null;
+    }
       
     public void setLocations(String[] locations) {
       this.locations = locations;

Modified: 
hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JobInProgress.java
URL: 
http://svn.apache.org/viewvc/hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JobInProgress.java?rev=677264&r1=677263&r2=677264&view=diff
==============================================================================
--- hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JobInProgress.java 
(original)
+++ hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JobInProgress.java 
Wed Jul 16 05:44:18 2008
@@ -1638,6 +1638,11 @@
         localJarFile = null;
       }
 
+      // clean up splits
+      for (int i = 0; i < maps.length; i++) {
+        maps[i].clearSplit();
+      }
+
       // JobClient always creates a new directory with job files
       // so we remove that directory to cleanup
       // Delete temp dfs dirs created if any, like in case of 

Modified: hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/MapTask.java
URL: 
http://svn.apache.org/viewvc/hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/MapTask.java?rev=677264&r1=677263&r2=677264&view=diff
==============================================================================
--- hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/MapTask.java 
(original)
+++ hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/MapTask.java Wed Jul 
16 05:44:18 2008
@@ -84,7 +84,7 @@
                  ) throws IOException {
     super(jobFile, taskId, partition);
     this.splitClass = splitClass;
-    this.split.set(split);
+    this.split = split;
   }
 
   @Override
@@ -114,6 +114,7 @@
     super.write(out);
     Text.writeString(out, splitClass);
     split.write(out);
+    split = null;
   }
   
   @Override

Modified: 
hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/TaskInProgress.java
URL: 
http://svn.apache.org/viewvc/hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/TaskInProgress.java?rev=677264&r1=677263&r2=677264&view=diff
==============================================================================
--- hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/TaskInProgress.java 
(original)
+++ hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/TaskInProgress.java 
Wed Jul 16 05:44:18 2008
@@ -777,6 +777,9 @@
    * Gets the Node list of input split locations sorted in rack order.
    */ 
   public String getSplitNodes() {
+    if ( rawSplit == null) {
+      return "";
+    }
     String[] splits = rawSplit.getLocations();
     Node[] nodes = new Node[splits.length];
     for (int i = 0; i < splits.length; i++) {
@@ -805,4 +808,8 @@
     return ret.toString();
   }
 
+  public void clearSplit() {
+    rawSplit.clearBytes();
+  }
+
 }