svn commit: r1056359 - in /nutch/branches/branch-1.3: CHANGES.txt src/plugin/protocol-http/src/java/org/apache/nutch/protocol/http/HttpResponse.java src/plugin/protocol-httpclient/src/java/org/apache/nutch/protocol/httpclient/HttpResponse.java

[jnioche]

Author: jnioche
Date: Fri Jan  7 15:46:25 2011
New Revision: 1056359

URL: http://svn.apache.org/viewvc?rev=1056359&view=rev
Log:
NUTCH-954 Strict application of Content-Length limit for http protocols (Alexis 
Detreglode via jnioche)

Modified:
    nutch/branches/branch-1.3/CHANGES.txt
    
nutch/branches/branch-1.3/src/plugin/protocol-http/src/java/org/apache/nutch/protocol/http/HttpResponse.java
    
nutch/branches/branch-1.3/src/plugin/protocol-httpclient/src/java/org/apache/nutch/protocol/httpclient/HttpResponse.java

Modified: nutch/branches/branch-1.3/CHANGES.txt
URL: 
http://svn.apache.org/viewvc/nutch/branches/branch-1.3/CHANGES.txt?rev=1056359&r1=1056358&r2=1056359&view=diff
==============================================================================
--- nutch/branches/branch-1.3/CHANGES.txt (original)
+++ nutch/branches/branch-1.3/CHANGES.txt Fri Jan  7 15:46:25 2011
@@ -2,6 +2,8 @@ Nutch Change Log
 
 Release 1.3 - Current Development
 
+* NUTCH-954 Strict application of Content-Length limit for http protocols 
(Alexis Detreglode via jnioche)
+
 * NUTCH-950 DomainURLFilter throws NPE on bogus urls (Alexis Detreglode via 
jnioche)
 
 * NUTCH-935 basicurlnormalizer removes unnecessary /./ in URLs

Modified: 
nutch/branches/branch-1.3/src/plugin/protocol-http/src/java/org/apache/nutch/protocol/http/HttpResponse.java
URL: 
http://svn.apache.org/viewvc/nutch/branches/branch-1.3/src/plugin/protocol-http/src/java/org/apache/nutch/protocol/http/HttpResponse.java?rev=1056359&r1=1056358&r2=1056359&view=diff
==============================================================================
--- 
nutch/branches/branch-1.3/src/plugin/protocol-http/src/java/org/apache/nutch/protocol/http/HttpResponse.java
 (original)
+++ 
nutch/branches/branch-1.3/src/plugin/protocol-http/src/java/org/apache/nutch/protocol/http/HttpResponse.java
 Fri Jan  7 15:46:25 2011
@@ -225,12 +225,10 @@ public class HttpResponse implements Res
     ByteArrayOutputStream out = new ByteArrayOutputStream(Http.BUFFER_SIZE);
     byte[] bytes = new byte[Http.BUFFER_SIZE];
     int length = 0;                           // read content
-    for (int i = in.read(bytes); i != -1; i = in.read(bytes)) {
+    for (int i = in.read(bytes); i != -1 && length + i <= contentLength; i = 
in.read(bytes)) {
 
       out.write(bytes, 0, i);
       length += i;
-      if (length >= contentLength)
-        break;
     }
     content = out.toByteArray();
   }

Modified: 
nutch/branches/branch-1.3/src/plugin/protocol-httpclient/src/java/org/apache/nutch/protocol/httpclient/HttpResponse.java
URL: 
http://svn.apache.org/viewvc/nutch/branches/branch-1.3/src/plugin/protocol-httpclient/src/java/org/apache/nutch/protocol/httpclient/HttpResponse.java?rev=1056359&r1=1056358&r2=1056359&view=diff
==============================================================================
--- 
nutch/branches/branch-1.3/src/plugin/protocol-httpclient/src/java/org/apache/nutch/protocol/httpclient/HttpResponse.java
 (original)
+++ 
nutch/branches/branch-1.3/src/plugin/protocol-httpclient/src/java/org/apache/nutch/protocol/httpclient/HttpResponse.java
 Fri Jan  7 15:46:25 2011
@@ -124,7 +124,7 @@ public class HttpResponse implements Res
         int totalRead = 0;
         ByteArrayOutputStream out = new ByteArrayOutputStream();
         while ((bufferFilled = in.read(buffer, 0, buffer.length)) != -1
-            && totalRead < contentLength) {
+            && totalRead + bufferFilled < contentLength) {
           totalRead += bufferFilled;
           out.write(buffer, 0, bufferFilled);
         }