Author: jerome
Date: Thu Mar 2 14:38:40 2006
New Revision: 382535
URL: http://svn.apache.org/viewcvs?rev=382535&view=rev
Log:
Fix content.limit inconsistency in http, ftp and file
Modified:
lucene/nutch/trunk/conf/nutch-default.xml
lucene/nutch/trunk/src/plugin/protocol-file/src/java/org/apache/nutch/protocol/file/FileResponse.java
lucene/nutch/trunk/src/plugin/protocol-ftp/src/java/org/apache/nutch/protocol/ftp/Client.java
Modified: lucene/nutch/trunk/conf/nutch-default.xml
URL:
http://svn.apache.org/viewcvs/lucene/nutch/trunk/conf/nutch-default.xml?rev=382535&r1=382534&r2=382535&view=diff
==============================================================================
--- lucene/nutch/trunk/conf/nutch-default.xml (original)
+++ lucene/nutch/trunk/conf/nutch-default.xml Thu Mar 2 14:38:40 2006
@@ -13,8 +13,8 @@
<name>file.content.limit</name>
<value>65536</value>
<description>The length limit for downloaded content, in bytes.
- If this value is larger than zero, content longer than it will be
- truncated; otherwise (zero or negative), no truncation at all.
+ If this value is nonnegative (>=0), content longer than it will be truncated;
+ otherwise, no truncation at all.
</description>
</property>
@@ -150,11 +150,11 @@
<name>ftp.content.limit</name>
<value>65536</value>
<description>The length limit for downloaded content, in bytes.
- If this value is larger than zero, content longer than it is truncated;
- otherwise (zero or negative), no truncation at all. Caution: classical
- ftp RFCs never defines partial transfer and, in fact, some ftp servers
- out there do not handle client side forced close-down very well.
- Our implementation tries its best to handle such situations smoothly.
+ If this value is nonnegative (>=0), content longer than it will be truncated;
+ otherwise, no truncation at all.
+ Caution: classical ftp RFCs never defines partial transfer and, in fact,
+ some ftp servers out there do not handle client side forced close-down very
+ well. Our implementation tries its best to handle such situations smoothly.
</description>
</property>
Modified:
lucene/nutch/trunk/src/plugin/protocol-file/src/java/org/apache/nutch/protocol/file/FileResponse.java
URL:
http://svn.apache.org/viewcvs/lucene/nutch/trunk/src/plugin/protocol-file/src/java/org/apache/nutch/protocol/file/FileResponse.java?rev=382535&r1=382534&r2=382535&view=diff
==============================================================================
---
lucene/nutch/trunk/src/plugin/protocol-file/src/java/org/apache/nutch/protocol/file/FileResponse.java
(original)
+++
lucene/nutch/trunk/src/plugin/protocol-file/src/java/org/apache/nutch/protocol/file/FileResponse.java
Thu Mar 2 14:38:40 2006
@@ -167,7 +167,7 @@
// capture content
int len = (int) size;
- if (this.file.maxContentLength > 0 && len > this.file.maxContentLength)
+ if (this.file.maxContentLength >= 0 && len > this.file.maxContentLength)
len = this.file.maxContentLength;
this.content = new byte[len];
Modified:
lucene/nutch/trunk/src/plugin/protocol-ftp/src/java/org/apache/nutch/protocol/ftp/Client.java
URL:
http://svn.apache.org/viewcvs/lucene/nutch/trunk/src/plugin/protocol-ftp/src/java/org/apache/nutch/protocol/ftp/Client.java?rev=382535&r1=382534&r2=382535&view=diff
==============================================================================
---
lucene/nutch/trunk/src/plugin/protocol-ftp/src/java/org/apache/nutch/protocol/ftp/Client.java
(original)
+++
lucene/nutch/trunk/src/plugin/protocol-ftp/src/java/org/apache/nutch/protocol/ftp/Client.java
Thu Mar 2 14:38:40 2006
@@ -344,9 +344,9 @@
}
entries.add(ftpFile);
count += line.length();
- // impose download limit if limit > 0, otherwise no limit
+ // impose download limit if limit >= 0, otherwise no limit
// here, cut off is up to the line when total bytes is just over limit
- if (limit > 0 && count > limit) {
+ if (limit >= 0 && count > limit) {
mandatory_close = true;
break;
}
@@ -409,9 +409,9 @@
new byte[org.apache.commons.net.io.Util.DEFAULT_COPY_BUFFER_SIZE];
while((len=input.read(buf,0,buf.length)) != -1){
count += len;
- // impose download limit if limit > 0, otherwise no limit
+ // impose download limit if limit >= 0, otherwise no limit
// here, cut off is exactly of limit bytes
- if (limit > 0 && count > limit) {
+ if (limit >= 0 && count > limit) {
os.write(buf,0,len-(count-limit));
mandatory_close = true;
break;
