Author: ab
Date: Wed Aug 11 13:33:44 2010
New Revision: 984401
URL: http://svn.apache.org/viewvc?rev=984401&view=rev
Log:
NUTCH-872 Change the default fetcher.parse to FALSE.
Modified:
nutch/trunk/CHANGES.txt
nutch/trunk/conf/nutch-default.xml
nutch/trunk/src/java/org/apache/nutch/fetcher/FetcherJob.java
nutch/trunk/src/java/org/apache/nutch/fetcher/FetcherReducer.java
Modified: nutch/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/nutch/trunk/CHANGES.txt?rev=984401&r1=984400&r2=984401&view=diff
==============================================================================
--- nutch/trunk/CHANGES.txt (original)
+++ nutch/trunk/CHANGES.txt Wed Aug 11 13:33:44 2010
@@ -2,6 +2,8 @@ Nutch Change Log
Release 2.0 - Current Development
+* NUTCH-872 Change the default fetcher.parse to FALSE (ab).
+
* NUTCH-861 Renamed HTMLParseFilter into ParseFilter
* NUTCH-876 Remove remaining robots/IP blocking code in lib-http (ab)
Modified: nutch/trunk/conf/nutch-default.xml
URL:
http://svn.apache.org/viewvc/nutch/trunk/conf/nutch-default.xml?rev=984401&r1=984400&r2=984401&view=diff
==============================================================================
--- nutch/trunk/conf/nutch-default.xml (original)
+++ nutch/trunk/conf/nutch-default.xml Wed Aug 11 13:33:44 2010
@@ -641,8 +641,9 @@
<property>
<name>fetcher.parse</name>
- <value>true</value>
- <description>If true, fetcher will parse content.</description>
+ <value>false</value>
+ <description>If true, fetcher will parse content. NOTE: previous releases
would
+ default to true. Since 2.0 this is set to false as a safer
default.</description>
</property>
<property>
Modified: nutch/trunk/src/java/org/apache/nutch/fetcher/FetcherJob.java
URL:
http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/fetcher/FetcherJob.java?rev=984401&r1=984400&r2=984401&view=diff
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/fetcher/FetcherJob.java (original)
+++ nutch/trunk/src/java/org/apache/nutch/fetcher/FetcherJob.java Wed Aug 11
13:33:44 2010
@@ -138,7 +138,17 @@ public class FetcherJob implements Tool
return fields;
}
- public int fetch(int threads, String crawlId, boolean shouldResume, boolean
isParsing)
+ /**
+ * Run fetcher.
+ * @param threads number of threads per map task
+ * @param crawlId crawlId (obtained from Generator) or null to fetch all
generated fetchlists
+ * @param shouldResume
+ * @param parse if true, then parse content immediately, if false then a
separate
+ * run of {...@link ParserJob} will be needed.
+ * @return 0 on success
+ * @throws Exception
+ */
+ public int fetch(int threads, String crawlId, boolean shouldResume, boolean
parse)
throws Exception {
LOG.info("FetcherJob: starting");
@@ -148,7 +158,7 @@ public class FetcherJob implements Tool
getConf().setInt(THREADS_KEY, threads);
}
getConf().set(GeneratorJob.CRAWL_ID, crawlId);
- getConf().setBoolean(PARSE_KEY, isParsing);
+ getConf().setBoolean(PARSE_KEY, parse);
getConf().setBoolean(RESUME_KEY, shouldResume);
// set the actual time for the timelimit relative
@@ -224,10 +234,10 @@ public class FetcherJob implements Tool
public int run(String[] args) throws Exception {
int threads = -1;
boolean shouldResume = false;
- boolean isParsing = true;
+ boolean parse = getConf().getBoolean(PARSE_KEY, false);
String crawlId;
- String usage = "Usage: FetcherJob (<crawl id> | -all) [-threads N]
[-noParsing] [-resume]";
+ String usage = "Usage: FetcherJob (<crawl id> | -all) [-threads N]
[-parse] [-resume]";
if (args.length == 0) {
System.err.println(usage);
@@ -235,7 +245,7 @@ public class FetcherJob implements Tool
}
crawlId = args[0];
- if (crawlId.equals("-threads") || crawlId.equals("-resume") ||
crawlId.equals("-noParsing")) {
+ if (crawlId.equals("-threads") || crawlId.equals("-resume") ||
crawlId.equals("-parse")) {
System.err.println(usage);
return -1;
}
@@ -245,12 +255,12 @@ public class FetcherJob implements Tool
threads = Integer.parseInt(args[++i]);
} else if ("-resume".equals(args[i])) {
shouldResume = true;
- } else if ("-noParsing".equals(args[i])) {
- isParsing = false;
+ } else if ("-parse".equals(args[i])) {
+ parse = true;
}
}
- int fetchcode = fetch(threads, crawlId, shouldResume, isParsing); // run
the Fetcher
+ int fetchcode = fetch(threads, crawlId, shouldResume, parse); // run the
Fetcher
return fetchcode;
}
Modified: nutch/trunk/src/java/org/apache/nutch/fetcher/FetcherReducer.java
URL:
http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/fetcher/FetcherReducer.java?rev=984401&r1=984400&r2=984401&view=diff
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/fetcher/FetcherReducer.java (original)
+++ nutch/trunk/src/java/org/apache/nutch/fetcher/FetcherReducer.java Wed Aug
11 13:33:44 2010
@@ -63,7 +63,7 @@ extends GoraReducer<IntWritable, FetchEn
private FetchItemQueues fetchQueues;
- private boolean isParsing;
+ private boolean parse;
private ParseUtil parseUtil;
@@ -590,7 +590,7 @@ extends GoraReducer<IntWritable, FetchEn
Mark.FETCH_MARK.putMark(fit.page,
Mark.GENERATE_MARK.checkMark(fit.page));
String key = TableUtil.reverseUrl(fit.url);
- if (isParsing) {
+ if (parse) {
URLWebPage redirectedPage = parseUtil.process(key, fit.page);
if (redirectedPage != null) {
context.write(TableUtil.reverseUrl(redirectedPage.getUrl()),
@@ -708,16 +708,12 @@ extends GoraReducer<IntWritable, FetchEn
Configuration conf = context.getConfiguration();
this.fetchQueues = new FetchItemQueues(conf);
int threadCount = conf.getInt("fetcher.threads.fetch", 10);
- isParsing = conf.getBoolean("fetcher.parse", true);
- if (isParsing) {
+ parse = conf.getBoolean(FetcherJob.PARSE_KEY, false);
+ if (parse) {
parseUtil = new ParseUtil(conf);
}
LOG.info("Fetcher: threads: " + threadCount);
- // set non-blocking & no-robots mode for HTTP protocol plugins.
- conf.setBoolean(Protocol.CHECK_BLOCKING, false);
- conf.setBoolean(Protocol.CHECK_ROBOTS, false);
-
feeder = new QueueFeeder(context, fetchQueues, threadCount * 50);
feeder.start();
