Author: ferdy
Date: Mon May 7 09:48:57 2012
New Revision: 1334945
URL: http://svn.apache.org/viewvc?rev=1334945&view=rev
Log:
NUTCH-1354 nutchgora support fetcher.queue.depth.multiplier property
Modified:
nutch/branches/nutchgora/CHANGES.txt
nutch/branches/nutchgora/conf/nutch-default.xml
nutch/branches/nutchgora/src/java/org/apache/nutch/fetcher/FetcherReducer.java
Modified: nutch/branches/nutchgora/CHANGES.txt
URL:
http://svn.apache.org/viewvc/nutch/branches/nutchgora/CHANGES.txt?rev=1334945&r1=1334944&r2=1334945&view=diff
==============================================================================
--- nutch/branches/nutchgora/CHANGES.txt (original)
+++ nutch/branches/nutchgora/CHANGES.txt Mon May 7 09:48:57 2012
@@ -1,6 +1,8 @@
Nutch Change Log
Release nutchgora - Current Development
+* NUTCH-1354 nutchgora support fetcher.queue.depth.multiplier property (ferdy)
+
* NUTCH-1353 nutchgora DomainStatistics support crawlId, counter bug and
reformatting (ferdy)
* NUTCH-1350 remove unused dependancy because of access restriction (ferdy)
Modified: nutch/branches/nutchgora/conf/nutch-default.xml
URL:
http://svn.apache.org/viewvc/nutch/branches/nutchgora/conf/nutch-default.xml?rev=1334945&r1=1334944&r2=1334945&view=diff
==============================================================================
--- nutch/branches/nutchgora/conf/nutch-default.xml (original)
+++ nutch/branches/nutchgora/conf/nutch-default.xml Mon May 7 09:48:57 2012
@@ -686,6 +686,16 @@
</description>
</property>
+<property>
+ <name>fetcher.queue.depth.multiplier</name>
+ <value>50</value>
+ <description>(EXPERT)The fetcher buffers the incoming URLs into queues based
on the [host|domain|IP]
+ (see param fetcher.queue.mode). The depth of the queue is the number of
threads times the value of this parameter.
+ A large value requires more memory but can improve the performance of the
fetch when the order of the URLS in the fetch list
+ is not optimal.
+ </description>
+</property>
+
<!-- indexingfilter plugin properties -->
<property>
Modified:
nutch/branches/nutchgora/src/java/org/apache/nutch/fetcher/FetcherReducer.java
URL:
http://svn.apache.org/viewvc/nutch/branches/nutchgora/src/java/org/apache/nutch/fetcher/FetcherReducer.java?rev=1334945&r1=1334944&r2=1334945&view=diff
==============================================================================
---
nutch/branches/nutchgora/src/java/org/apache/nutch/fetcher/FetcherReducer.java
(original)
+++
nutch/branches/nutchgora/src/java/org/apache/nutch/fetcher/FetcherReducer.java
Mon May 7 09:48:57 2012
@@ -775,7 +775,8 @@ extends GoraReducer<IntWritable, FetchEn
}
LOG.info("Fetcher: threads: " + threadCount);
- feeder = new QueueFeeder(context, fetchQueues, threadCount * 50);
+ int maxFeedPerThread = conf.getInt("fetcher.queue.depth.multiplier", 50);
+ feeder = new QueueFeeder(context, fetchQueues, threadCount *
maxFeedPerThread);
feeder.start();
for (int i = 0; i < threadCount; i++) { // spawn threads
