svn commit: r1077170 - in /hadoop/common/branches/branch-0.20-security-patches/src: core/org/apache/hadoop/ipc/Server.java test/org/apache/hadoop/ipc/TestIPCServerResponder.java

[omalley]

Author: omalley
Date: Fri Mar  4 03:48:20 2011
New Revision: 1077170

URL: http://svn.apache.org/viewvc?rev=1077170&view=rev
Log:
commit 3e5b27eb421558496e8071b440e7deea29d9d3fa
Author: Suresh Srinivas <sures...@yahoo-inc.com>
Date:   Thu Dec 24 00:33:33 2009 -0800

    HADOOP-6460 from 
https://issues.apache.org/jira/secure/attachment/12428898/hadoop-6460.rel20.patch
    
    +++ b/YAHOO-CHANGES.txt
    +    HADOOP-6460. Reinitializes buffers used for serializing responses in 
ipc
    +    server on exceeding maximum response size to free up Java heap. 
(suresh)

Modified:
    
hadoop/common/branches/branch-0.20-security-patches/src/core/org/apache/hadoop/ipc/Server.java
    
hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/ipc/TestIPCServerResponder.java

Modified: 
hadoop/common/branches/branch-0.20-security-patches/src/core/org/apache/hadoop/ipc/Server.java
URL: 
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/core/org/apache/hadoop/ipc/Server.java?rev=1077170&r1=1077169&r2=1077170&view=diff
==============================================================================
--- 
hadoop/common/branches/branch-0.20-security-patches/src/core/org/apache/hadoop/ipc/Server.java
 (original)
+++ 
hadoop/common/branches/branch-0.20-security-patches/src/core/org/apache/hadoop/ipc/Server.java
 Fri Mar  4 03:48:20 2011
@@ -100,6 +100,12 @@ public abstract class Server {
    */
   private static final int MAX_QUEUE_SIZE_PER_HANDLER = 100;
   
+  /**
+   * Initial and max size of response buffer
+   */
+  static int INITIAL_RESP_BUF_SIZE = 10240;
+  static int MAX_RESP_BUF_SIZE = 1024*1024;
+    
   public static final Log LOG = LogFactory.getLog(Server.class);
 
   private static final ThreadLocal<Server> SERVER = new ThreadLocal<Server>();
@@ -1146,7 +1152,8 @@ public abstract class Server {
     public void run() {
       LOG.info(getName() + ": starting");
       SERVER.set(Server.this);
-      ByteArrayOutputStream buf = new ByteArrayOutputStream(10240);
+      ByteArrayOutputStream buf = 
+        new ByteArrayOutputStream(INITIAL_RESP_BUF_SIZE);
       while (running) {
         try {
           final Call call = callQueue.take(); // pop the queue; maybe blocked 
here
@@ -1186,10 +1193,16 @@ public abstract class Server {
             error = StringUtils.stringifyException(e);
           }
           CurCall.set(null);
-
           setupResponse(buf, call, 
                         (error == null) ? Status.SUCCESS : Status.ERROR, 
                         value, errorClass, error);
+          // Discard the large buf and reset it back to 
+          // smaller size to freeup heap
+          if (buf.size() > MAX_RESP_BUF_SIZE) {
+            LOG.warn("Large response size " + buf.size() + " for call " + 
+                call.toString());
+            buf = new ByteArrayOutputStream(INITIAL_RESP_BUF_SIZE);
+          }
           responder.doRespond(call);
         } catch (InterruptedException e) {
           if (running) {                          // unexpected -- log it

Modified: 
hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/ipc/TestIPCServerResponder.java
URL: 
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/ipc/TestIPCServerResponder.java?rev=1077170&r1=1077169&r2=1077170&view=diff
==============================================================================
--- 
hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/ipc/TestIPCServerResponder.java
 (original)
+++ 
hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/ipc/TestIPCServerResponder.java
 Fri Mar  4 03:48:20 2011
@@ -114,6 +114,12 @@ public class TestIPCServerResponder exte
     }
   }
 
+  public void testResponseBuffer() throws Exception {
+    Server.INITIAL_RESP_BUF_SIZE = 1;
+    Server.MAX_RESP_BUF_SIZE = 1;
+    testServerResponder(1, true, 1, 1, 5);
+  }
+
   public void testServerResponder() throws Exception {
     testServerResponder(10, true, 1, 10, 200);
   }