Author: jnioche
Date: Sat Oct 1 08:24:32 2011
New Revision: 1177944
URL: http://svn.apache.org/viewvc?rev=1177944&view=rev
Log:
NUTCH-1141 Configurable Fetcher queue depth
Modified:
nutch/trunk/CHANGES.txt
nutch/trunk/conf/nutch-default.xml
nutch/trunk/src/java/org/apache/nutch/fetcher/Fetcher.java
Modified: nutch/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/nutch/trunk/CHANGES.txt?rev=1177944&r1=1177943&r2=1177944&view=diff
==============================================================================
--- nutch/trunk/CHANGES.txt (original)
+++ nutch/trunk/CHANGES.txt Sat Oct 1 08:24:32 2011
@@ -2,6 +2,8 @@ Nutch Change Log
Release 1.4 - Current development
+* NUTCH-1141 Configurable Fetcher queue depth (jnioche)
+
* NUTCH-1091 Remove commons logging dependency from Nutch branch and trunk
(lewismc)
* NUTCH-672 allow unit tests to be run from bin/nutch (Todd Lipton via lewismc)
Modified: nutch/trunk/conf/nutch-default.xml
URL:
http://svn.apache.org/viewvc/nutch/trunk/conf/nutch-default.xml?rev=1177944&r1=1177943&r2=1177944&view=diff
==============================================================================
--- nutch/trunk/conf/nutch-default.xml (original)
+++ nutch/trunk/conf/nutch-default.xml Sat Oct 1 08:24:32 2011
@@ -722,6 +722,16 @@
</description>
</property>
+<property>
+ <name>fetcher.queue.depth.multiplier</name>
+ <value>50</value>
+ <description>(EXPERT)The fetcher buffers the incoming URLs into queues based
on the [host|domain|IP]
+ (see param fetcher.queue.mode). The depth of the queue is the number of
threads times the value of this parameter.
+ A large value requires more memory but can improve the performance of the
fetch when the order of the URLS in the fetch list
+ is not optimal.
+ </description>
+</property>
+
<!-- moreindexingfilter plugin properties -->
<property>
Modified: nutch/trunk/src/java/org/apache/nutch/fetcher/Fetcher.java
URL:
http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/fetcher/Fetcher.java?rev=1177944&r1=1177943&r2=1177944&view=diff
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/fetcher/Fetcher.java (original)
+++ nutch/trunk/src/java/org/apache/nutch/fetcher/Fetcher.java Sat Oct 1
08:24:32 2011
@@ -1057,7 +1057,9 @@ public class Fetcher extends Configured
int timeoutDivisor = getConf().getInt("fetcher.threads.timeout.divisor",
2);
if (LOG.isInfoEnabled()) { LOG.info("Fetcher: time-out divisor: " +
timeoutDivisor); }
- feeder = new QueueFeeder(input, fetchQueues, threadCount * 50);
+ int queueDepthMuliplier =
getConf().getInt("fetcher.queue.depth.multiplier", 50);
+
+ feeder = new QueueFeeder(input, fetchQueues, threadCount *
queueDepthMuliplier);
//feeder.setPriority((Thread.MAX_PRIORITY + Thread.NORM_PRIORITY) / 2);
// the value of the time limit is either -1 or the time where it should
finish
