svn commit: r1587955 - in /hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs: ./ src/main/java/org/apache/hadoop/hdfs/server/datanode/web/resources/

[daryn]

Author: daryn
Date: Wed Apr 16 16:06:28 2014
New Revision: 1587955

URL: http://svn.apache.org/r1587955
Log:
svn merge -c 1587954 FIXES: HDFS-6214.  Webhdfs has poor throughput for files 
>2GB (daryn)

Modified:
    hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
    
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/resources/DatanodeWebHdfsMethods.java
    
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/resources/OpenEntity.java

Modified: 
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL: 
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1587955&r1=1587954&r2=1587955&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt 
(original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt 
Wed Apr 16 16:06:28 2014
@@ -51,6 +51,8 @@ Release 2.5.0 - UNRELEASED
 
   OPTIMIZATIONS
 
+    HDFS-6214. Webhdfs has poor throughput for files >2GB (daryn)
+
   BUG FIXES 
 
     HDFS-6112. NFS Gateway docs are incorrect for allowed hosts configuration.

Modified: 
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/resources/DatanodeWebHdfsMethods.java
URL: 
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/resources/DatanodeWebHdfsMethods.java?rev=1587955&r1=1587954&r2=1587955&view=diff
==============================================================================
--- 
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/resources/DatanodeWebHdfsMethods.java
 (original)
+++ 
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/resources/DatanodeWebHdfsMethods.java
 Wed Apr 16 16:06:28 2014
@@ -430,10 +430,18 @@ public class DatanodeWebHdfsMethods {
         Math.min(length.getValue(), in.getVisibleLength() - offset.getValue()) 
:
         in.getVisibleLength() - offset.getValue();
 
+      // jetty 6 reserves 12 bytes in the out buffer for chunked responses
+      // (file length > 2GB) which causes extremely poor performance when
+      // 12 bytes of the output spill into another buffer which results
+      // in a big and little write
+      int outBufferSize = response.getBufferSize();
+      if (n > Integer.MAX_VALUE) {
+        outBufferSize -= 12;
+      }
       /**
        * Allow the Web UI to perform an AJAX request to get the data.
        */
-      return Response.ok(new OpenEntity(in, n, dfsclient))
+      return Response.ok(new OpenEntity(in, n, outBufferSize, dfsclient))
           .type(MediaType.APPLICATION_OCTET_STREAM)
           .header("Access-Control-Allow-Methods", "GET")
           .header("Access-Control-Allow-Origin", "*")

Modified: 
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/resources/OpenEntity.java
URL: 
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/resources/OpenEntity.java?rev=1587955&r1=1587954&r2=1587955&view=diff
==============================================================================
--- 
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/resources/OpenEntity.java
 (original)
+++ 
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/resources/OpenEntity.java
 Wed Apr 16 16:06:28 2014
@@ -37,12 +37,14 @@ import org.apache.hadoop.io.IOUtils;
 public class OpenEntity {
   private final HdfsDataInputStream in;
   private final long length;
+  private final int outBufferSize;
   private final DFSClient dfsclient;
   
   OpenEntity(final HdfsDataInputStream in, final long length,
-      final DFSClient dfsclient) {
+      final int outBufferSize, final DFSClient dfsclient) {
     this.in = in;
     this.length = length;
+    this.outBufferSize = outBufferSize;
     this.dfsclient = dfsclient;
   }
   
@@ -71,7 +73,17 @@ public class OpenEntity {
         MultivaluedMap<String, Object> httpHeaders, OutputStream out
         ) throws IOException {
       try {
-        IOUtils.copyBytes(e.in, out, e.length, false);
+        byte[] buf = new byte[e.outBufferSize];
+        long remaining = e.length;
+        while (remaining > 0) {
+          int read = e.in.read(buf, 0, (int)Math.min(buf.length, remaining));
+          if (read == -1) { // EOF
+            break;
+          }
+          out.write(buf, 0, read);
+          out.flush();
+          remaining -= read;
+        }
       } finally {
         IOUtils.cleanup(DatanodeWebHdfsMethods.LOG, e.in);
         IOUtils.cleanup(DatanodeWebHdfsMethods.LOG, e.dfsclient);