Author: ab
Date: Tue Apr 24 14:32:51 2007
New Revision: 532088
URL: http://svn.apache.org/viewvc?view=rev&rev=532088
Log:
NUTCH-474 - Fix crawlDelay and blocking checks.
Modified:
lucene/nutch/trunk/CHANGES.txt
lucene/nutch/trunk/src/java/org/apache/nutch/fetcher/Fetcher2.java
Modified: lucene/nutch/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/lucene/nutch/trunk/CHANGES.txt?view=diff&rev=532088&r1=532087&r2=532088
==============================================================================
--- lucene/nutch/trunk/CHANGES.txt (original)
+++ lucene/nutch/trunk/CHANGES.txt Tue Apr 24 14:32:51 2007
@@ -2,6 +2,9 @@
Unreleased changes (1.0-dev)
+ 1. NUTCH-474 - Fetcher2 crawlDelay and blocking fix (Dogacan Guney via ab)
+
+
Release 0.9 - 2007-04-02
Modified: lucene/nutch/trunk/src/java/org/apache/nutch/fetcher/Fetcher2.java
URL:
http://svn.apache.org/viewvc/lucene/nutch/trunk/src/java/org/apache/nutch/fetcher/Fetcher2.java?view=diff&rev=532088&r1=532087&r2=532088
==============================================================================
--- lucene/nutch/trunk/src/java/org/apache/nutch/fetcher/Fetcher2.java
(original)
+++ lucene/nutch/trunk/src/java/org/apache/nutch/fetcher/Fetcher2.java Tue Apr
24 14:32:51 2007
@@ -236,7 +236,7 @@
public FetchItem getFetchItem() {
if (inProgress.size() >= maxThreads) return null;
long now = System.currentTimeMillis();
- long last = endTime.get() + (maxThreads > 1 ? crawlDelay :
minCrawlDelay);
+ long last = endTime.get() + (maxThreads > 1 ? minCrawlDelay :
crawlDelay);
if (last > now) return null;
FetchItem it = null;
if (queue.size() == 0) return null;
@@ -771,8 +771,8 @@
feeder.start();
// set non-blocking & no-robots mode for HTTP protocol plugins.
- getConf().setBoolean("http.plugin.check.blocking", false);
- getConf().setBoolean("http.plugin.check.robots", false);
+ getConf().setBoolean(Protocol.CHECK_BLOCKING, false);
+ getConf().setBoolean(Protocol.CHECK_ROBOTS, false);
for (int i = 0; i < threadCount; i++) { // spawn threads
new FetcherThread(getConf()).start();
-------------------------------------------------------------------------
This SF.net email is sponsored by DB2 Express
Download DB2 Express C - the FREE version of DB2 express and take
control of your XML. No limits. Just data. Click to get it now.
http://sourceforge.net/powerbar/db2/
_______________________________________________
Nutch-cvs mailing list
Nutch-cvs@lists.sourceforge.net
https://lists.sourceforge.net/lists/listinfo/nutch-cvs
