svn commit: r884290 - in /hadoop/common/branches/branch-0.20: CHANGES.txt src/mapred/org/apache/hadoop/mapred/ReduceTask.java

[cdouglas]

Author: cdouglas
Date: Wed Nov 25 21:57:18 2009
New Revision: 884290

URL: http://svn.apache.org/viewvc?rev=884290&view=rev
Log:
MAPREDUCE-1182. Fix overflow in reduce causing allocations to exceed the
configured threshold.

Modified:
    hadoop/common/branches/branch-0.20/CHANGES.txt
    
hadoop/common/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/ReduceTask.java

Modified: hadoop/common/branches/branch-0.20/CHANGES.txt
URL: 
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20/CHANGES.txt?rev=884290&r1=884289&r2=884290&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20/CHANGES.txt (original)
+++ hadoop/common/branches/branch-0.20/CHANGES.txt Wed Nov 25 21:57:18 2009
@@ -72,6 +72,9 @@
     HADOOP-6269. Fix threading issue with defaultResource in Configuration.
     (Sreekanth Ramakrishnan via cdouglas)
 
+    MAPREDUCE-1182. Fix overflow in reduce causing allocations to exceed the
+    configured threshold. (cdouglas)
+
 Release 0.20.1 - 2009-09-01
 
   INCOMPATIBLE CHANGES

Modified: 
hadoop/common/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/ReduceTask.java
URL: 
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/ReduceTask.java?rev=884290&r1=884289&r2=884290&view=diff
==============================================================================
--- 
hadoop/common/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/ReduceTask.java
 (original)
+++ 
hadoop/common/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/ReduceTask.java
 Wed Nov 25 21:57:18 2009
@@ -968,13 +968,13 @@
        * simultaneously after which a merge is triggered. */ 
       private static final float MAX_STALLED_SHUFFLE_THREADS_FRACTION = 0.75f;
       
-      private final int maxSize;
-      private final int maxSingleShuffleLimit;
+      private final long maxSize;
+      private final long maxSingleShuffleLimit;
       
-      private int size = 0;
+      private long size = 0;
       
       private Object dataAvailable = new Object();
-      private int fullSize = 0;
+      private long fullSize = 0;
       private int numPendingRequests = 0;
       private int numRequiredMapOutputs = 0;
       private int numClosed = 0;
@@ -987,10 +987,10 @@
           throw new IOException("mapred.job.shuffle.input.buffer.percent" +
                                 maxInMemCopyUse);
         }
-        maxSize = (int)Math.min(
+        maxSize = (long)Math.min(
             Runtime.getRuntime().maxMemory() * maxInMemCopyUse,
             Integer.MAX_VALUE);
-        maxSingleShuffleLimit = (int)(maxSize * 
MAX_SINGLE_SHUFFLE_SEGMENT_FRACTION);
+        maxSingleShuffleLimit = (long)(maxSize * 
MAX_SINGLE_SHUFFLE_SEGMENT_FRACTION);
         LOG.info("ShuffleRamManager: MemoryLimit=" + maxSize + 
                  ", MaxSingleShuffleLimit=" + maxSingleShuffleLimit);
       }
@@ -1102,10 +1102,6 @@
         return (float)fullSize/maxSize;
       }
 
-      int getMemoryLimit() {
-        return maxSize;
-      }
-      
       boolean canFitInMemory(long requestedSize) {
         return (requestedSize < Integer.MAX_VALUE && 
                 requestedSize < maxSingleShuffleLimit);