Author: suresh
Date: Thu Dec 24 00:50:08 2009
New Revision: 893667
URL: http://svn.apache.org/viewvc?rev=893667&view=rev
Log:
HADOOP-6460. Merging change 893666 from trunk to 21 -Reinitializes buffers used
for serializing responses in ipc server on exceeding maximum response size to
free up Java heap. Contributed by Suresh Srinivas.
Modified:
hadoop/common/branches/branch-0.21/CHANGES.txt
hadoop/common/branches/branch-0.21/src/java/org/apache/hadoop/ipc/Server.java
hadoop/common/branches/branch-0.21/src/test/core/org/apache/hadoop/ipc/TestIPCServerResponder.java
Modified: hadoop/common/branches/branch-0.21/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.21/CHANGES.txt?rev=893667&r1=893666&r2=893667&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.21/CHANGES.txt (original)
+++ hadoop/common/branches/branch-0.21/CHANGES.txt Thu Dec 24 00:50:08 2009
@@ -1161,6 +1161,10 @@
HADOOP-6269. Fix threading issue with defaultResource in Configuration.
(Sreekanth Ramakrishnan via cdouglas)
+ HADOOP-6460. Reinitializes buffers used for serializing responses in ipc
+ server on exceeding maximum response size to free up Java heap. (suresh)
+
+
Release 0.20.1 - 2009-09-01
INCOMPATIBLE CHANGES
Modified:
hadoop/common/branches/branch-0.21/src/java/org/apache/hadoop/ipc/Server.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.21/src/java/org/apache/hadoop/ipc/Server.java?rev=893667&r1=893666&r2=893667&view=diff
==============================================================================
---
hadoop/common/branches/branch-0.21/src/java/org/apache/hadoop/ipc/Server.java
(original)
+++
hadoop/common/branches/branch-0.21/src/java/org/apache/hadoop/ipc/Server.java
Thu Dec 24 00:50:08 2009
@@ -90,6 +90,12 @@
*/
private static final int MAX_QUEUE_SIZE_PER_HANDLER = 100;
+ /**
+ * Initial and max size of response buffer
+ */
+ static int INITIAL_RESP_BUF_SIZE = 10240;
+ static int MAX_RESP_BUF_SIZE = 1024*1024;
+
public static final Log LOG = LogFactory.getLog(Server.class);
private static final ThreadLocal<Server> SERVER = new ThreadLocal<Server>();
@@ -938,7 +944,8 @@
public void run() {
LOG.info(getName() + ": starting");
SERVER.set(Server.this);
- ByteArrayOutputStream buf = new ByteArrayOutputStream(10240);
+ ByteArrayOutputStream buf =
+ new ByteArrayOutputStream(INITIAL_RESP_BUF_SIZE);
while (running) {
try {
final Call call = callQueue.take(); // pop the queue; maybe blocked
here
@@ -979,10 +986,16 @@
error = StringUtils.stringifyException(e);
}
CurCall.set(null);
-
setupResponse(buf, call,
(error == null) ? Status.SUCCESS : Status.ERROR,
value, errorClass, error);
+ // Discard the large buf and reset it back to
+ // smaller size to freeup heap
+ if (buf.size() > MAX_RESP_BUF_SIZE) {
+ LOG.warn("Large response size " + buf.size() + " for call " +
+ call.toString());
+ buf = new ByteArrayOutputStream(INITIAL_RESP_BUF_SIZE);
+ }
responder.doRespond(call);
} catch (InterruptedException e) {
if (running) { // unexpected -- log it
Modified:
hadoop/common/branches/branch-0.21/src/test/core/org/apache/hadoop/ipc/TestIPCServerResponder.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.21/src/test/core/org/apache/hadoop/ipc/TestIPCServerResponder.java?rev=893667&r1=893666&r2=893667&view=diff
==============================================================================
---
hadoop/common/branches/branch-0.21/src/test/core/org/apache/hadoop/ipc/TestIPCServerResponder.java
(original)
+++
hadoop/common/branches/branch-0.21/src/test/core/org/apache/hadoop/ipc/TestIPCServerResponder.java
Thu Dec 24 00:50:08 2009
@@ -114,6 +114,12 @@
}
}
+ public void testResponseBuffer() throws Exception {
+ Server.INITIAL_RESP_BUF_SIZE = 1;
+ Server.MAX_RESP_BUF_SIZE = 1;
+ testServerResponder(1, true, 1, 1, 5);
+ }
+
public void testServerResponder() throws Exception {
testServerResponder(10, true, 1, 10, 200);
}
