svn commit: r1077060 - /hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/ReduceTask.java

[omalley]

Author: omalley
Date: Fri Mar  4 03:36:37 2011
New Revision: 1077060

URL: http://svn.apache.org/viewvc?rev=1077060&view=rev
Log:
commit 38b77ce8cd5005ca4634c77ff0b6e579bf73a2a8
Author: Chris Douglas <cdoug...@apache.org>
Date:   Wed Nov 25 14:41:33 2009 -0800

    MAPREDUCE:1182 from 
https://issues.apache.org/jira/secure/attachment/12424116/M1182-1v20.patch
    
    +++ b/YAHOO-CHANGES.txt
    +    MAPREDUCE-1182. Fix overflow in reduce causing allocations to exceed 
the
    +    configured threshold. (cdouglas)
    +

Modified:
    
hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/ReduceTask.java

Modified: 
hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/ReduceTask.java
URL: 
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/ReduceTask.java?rev=1077060&r1=1077059&r2=1077060&view=diff
==============================================================================
--- 
hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/ReduceTask.java
 (original)
+++ 
hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/ReduceTask.java
 Fri Mar  4 03:36:37 2011
@@ -988,13 +988,13 @@ class ReduceTask extends Task {
        * simultaneously after which a merge is triggered. */ 
       private static final float MAX_STALLED_SHUFFLE_THREADS_FRACTION = 0.75f;
       
-      private final int maxSize;
-      private final int maxSingleShuffleLimit;
+      private final long maxSize;
+      private final long maxSingleShuffleLimit;
       
-      private int size = 0;
+      private long size = 0;
       
       private Object dataAvailable = new Object();
-      private int fullSize = 0;
+      private long fullSize = 0;
       private int numPendingRequests = 0;
       private int numRequiredMapOutputs = 0;
       private int numClosed = 0;
@@ -1007,10 +1007,10 @@ class ReduceTask extends Task {
           throw new IOException("mapred.job.shuffle.input.buffer.percent" +
                                 maxInMemCopyUse);
         }
-        maxSize = (int)Math.min(
+        maxSize = (long)Math.min(
             Runtime.getRuntime().maxMemory() * maxInMemCopyUse,
             Integer.MAX_VALUE);
-        maxSingleShuffleLimit = (int)(maxSize * 
MAX_SINGLE_SHUFFLE_SEGMENT_FRACTION);
+        maxSingleShuffleLimit = (long)(maxSize * 
MAX_SINGLE_SHUFFLE_SEGMENT_FRACTION);
         LOG.info("ShuffleRamManager: MemoryLimit=" + maxSize + 
                  ", MaxSingleShuffleLimit=" + maxSingleShuffleLimit);
       }
@@ -1122,10 +1122,6 @@ class ReduceTask extends Task {
         return (float)fullSize/maxSize;
       }
 
-      int getMemoryLimit() {
-        return maxSize;
-      }
-      
       boolean canFitInMemory(long requestedSize) {
         return (requestedSize < Integer.MAX_VALUE && 
                 requestedSize < maxSingleShuffleLimit);