Author: talat
Date: Sat May 3 15:26:38 2014
New Revision: 1592218
URL: http://svn.apache.org/r1592218
Log:
NUTCH-1618 Turn speculative execution off for Fetching (talat)
Modified:
nutch/branches/2.x/CHANGES.txt
nutch/branches/2.x/src/java/org/apache/nutch/fetcher/FetcherJob.java
Modified: nutch/branches/2.x/CHANGES.txt
URL:
http://svn.apache.org/viewvc/nutch/branches/2.x/CHANGES.txt?rev=1592218&r1=1592217&r2=1592218&view=diff
==============================================================================
--- nutch/branches/2.x/CHANGES.txt (original)
+++ nutch/branches/2.x/CHANGES.txt Sat May 3 15:26:38 2014
@@ -2,6 +2,8 @@ Nutch Change Log
Current Development
+* NUTCH-1618 Turn speculative execution off for Fetching (talat)
+
* NUTCH-1657 ORIGINAL_CHAR_ENCODING and CHAR_ENCODING_FOR_CONVERSION never set
in HTMLParser (talat)
* NUTCH-1725 CleaningJob's reducer does not commit deleted docs. (ilhamikalkan
via talat)
Modified: nutch/branches/2.x/src/java/org/apache/nutch/fetcher/FetcherJob.java
URL:
http://svn.apache.org/viewvc/nutch/branches/2.x/src/java/org/apache/nutch/fetcher/FetcherJob.java?rev=1592218&r1=1592217&r2=1592218&view=diff
==============================================================================
--- nutch/branches/2.x/src/java/org/apache/nutch/fetcher/FetcherJob.java
(original)
+++ nutch/branches/2.x/src/java/org/apache/nutch/fetcher/FetcherJob.java Sat
May 3 15:26:38 2014
@@ -183,6 +183,10 @@ public class FetcherJob extends NutchToo
LOG.info("FetcherJob : timelimit set for : " +
getConf().getLong("fetcher.timelimit", -1));
numJobs = 1;
currentJob = new NutchJob(getConf(), "fetch");
+
+ // for politeness, don't permit parallel execution of a single task
+ currentJob.setReduceSpeculativeExecution(false);
+
Collection<WebPage.Field> fields = getFields(currentJob);
StorageUtils.initMapperJob(currentJob, fields, IntWritable.class,
FetchEntry.class, FetcherMapper.class, FetchEntryPartitioner.class,
false);
