Author: mattf
Date: Tue Jul 24 18:31:11 2012
New Revision: 1365209
URL: http://svn.apache.org/viewvc?rev=1365209&view=rev
Log:
MAPREDUCE-4399. Change the Jetty response buffer size to improve shuffle
performance. Contributed by Luke Lu.
Modified:
hadoop/common/branches/branch-1.0/CHANGES.txt (contents, props changed)
hadoop/common/branches/branch-1.0/src/mapred/org/apache/hadoop/mapred/TaskTracker.java
Modified: hadoop/common/branches/branch-1.0/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-1.0/CHANGES.txt?rev=1365209&r1=1365208&r2=1365209&view=diff
==============================================================================
--- hadoop/common/branches/branch-1.0/CHANGES.txt (original)
+++ hadoop/common/branches/branch-1.0/CHANGES.txt Tue Jul 24 18:31:11 2012
@@ -9,6 +9,9 @@ Release 1.0.4 - Unreleased
HADOOP-7154. Should set MALLOC_ARENA_MAX in hadoop-env.sh
(todd via mattf)
+ MAPREDUCE-4399. Change the Jetty response buffer size to improve
+ shuffle performance. (Luke Lu via suresh)
+
BUG FIXES
HDFS-3652. FSEditLog failure removes the wrong edit stream when storage
Propchange: hadoop/common/branches/branch-1.0/CHANGES.txt
------------------------------------------------------------------------------
Merged /hadoop/common/branches/branch-1.1/CHANGES.txt:r1365208
Modified:
hadoop/common/branches/branch-1.0/src/mapred/org/apache/hadoop/mapred/TaskTracker.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-1.0/src/mapred/org/apache/hadoop/mapred/TaskTracker.java?rev=1365209&r1=1365208&r2=1365209&view=diff
==============================================================================
---
hadoop/common/branches/branch-1.0/src/mapred/org/apache/hadoop/mapred/TaskTracker.java
(original)
+++
hadoop/common/branches/branch-1.0/src/mapred/org/apache/hadoop/mapred/TaskTracker.java
Tue Jul 24 18:31:11 2012
@@ -3790,7 +3790,8 @@ public class TaskTracker implements MRCo
public static class MapOutputServlet extends HttpServlet {
private static final long serialVersionUID = 1L;
private static final int MAX_BYTES_TO_READ = 64 * 1024;
-
+ // work around jetty internal buffering issues
+ private static final int RESPONSE_BUFFER_SIZE = MAX_BYTES_TO_READ + 16;
private static LRUCache<String, Path> fileCache = new LRUCache<String,
Path>(FILE_CACHE_SIZE);
private static LRUCache<String, Path> fileIndexCache = new
LRUCache<String, Path>(FILE_CACHE_SIZE);
@@ -3892,10 +3893,9 @@ public class TaskTracker implements MRCo
//set the custom "for-reduce-task" http header to the reduce task
number
//for which this map output is being transferred
response.setHeader(FOR_REDUCE_TASK, Integer.toString(reduce));
-
- //use the same buffersize as used for reading the data from disk
- response.setBufferSize(MAX_BYTES_TO_READ);
-
+
+ response.setBufferSize(RESPONSE_BUFFER_SIZE);
+
/**
* Read the data from the sigle map-output file and
* send it to the reducer.
