Author: ferdy
Date: Wed Nov 2 09:26:44 2011
New Revision: 1196516
URL: http://svn.apache.org/viewvc?rev=1196516&view=rev
Log:
NUTCH-1191 Port NUTCH-1102 to nutchgora - consistent use of fetcher.parse
Modified:
nutch/branches/nutchgora/CHANGES.txt
nutch/branches/nutchgora/src/java/org/apache/nutch/crawl/Crawler.java
nutch/branches/nutchgora/src/java/org/apache/nutch/fetcher/FetcherJob.java
nutch/branches/nutchgora/src/java/org/apache/nutch/metadata/Nutch.java
nutch/branches/nutchgora/src/java/org/apache/nutch/tools/Benchmark.java
Modified: nutch/branches/nutchgora/CHANGES.txt
URL:
http://svn.apache.org/viewvc/nutch/branches/nutchgora/CHANGES.txt?rev=1196516&r1=1196515&r2=1196516&view=diff
==============================================================================
--- nutch/branches/nutchgora/CHANGES.txt (original)
+++ nutch/branches/nutchgora/CHANGES.txt Wed Nov 2 09:26:44 2011
@@ -2,6 +2,8 @@ Nutch Change Log
Release nutchgora - Current Development
+* NUTCH-1191 Port NUTCH-1102 to nutchgora - consistent use of fetcher.parse
(ferdy)
+
* NUTCH-1187 Port NUTCH-1028 to nutchgora - log parser keys (ferdy)
* NUTCH-902 Add all necessary files and configuration so that nutch can be
used with different backends out-of-the-box (lewismc)
Modified: nutch/branches/nutchgora/src/java/org/apache/nutch/crawl/Crawler.java
URL:
http://svn.apache.org/viewvc/nutch/branches/nutchgora/src/java/org/apache/nutch/crawl/Crawler.java?rev=1196516&r1=1196515&r2=1196516&view=diff
==============================================================================
--- nutch/branches/nutchgora/src/java/org/apache/nutch/crawl/Crawler.java
(original)
+++ nutch/branches/nutchgora/src/java/org/apache/nutch/crawl/Crawler.java Wed
Nov 2 09:26:44 2011
@@ -119,10 +119,7 @@ public class Crawler extends NutchTool i
}
Integer depth = (Integer)args.get(Nutch.ARG_DEPTH);
if (depth == null) depth = 1;
- Boolean parse = (Boolean)args.get(Nutch.ARG_PARSE);
- if (parse == null) {
- parse = getConf().getBoolean(FetcherJob.PARSE_KEY, false);
- }
+ boolean parse = getConf().getBoolean(FetcherJob.PARSE_KEY, false);
String solrUrl = (String)args.get(Nutch.ARG_SOLR);
int onePhase = 3;
if (!parse) onePhase++;
Modified:
nutch/branches/nutchgora/src/java/org/apache/nutch/fetcher/FetcherJob.java
URL:
http://svn.apache.org/viewvc/nutch/branches/nutchgora/src/java/org/apache/nutch/fetcher/FetcherJob.java?rev=1196516&r1=1196515&r2=1196516&view=diff
==============================================================================
--- nutch/branches/nutchgora/src/java/org/apache/nutch/fetcher/FetcherJob.java
(original)
+++ nutch/branches/nutchgora/src/java/org/apache/nutch/fetcher/FetcherJob.java
Wed Nov 2 09:26:44 2011
@@ -137,7 +137,7 @@ public class FetcherJob extends NutchToo
public Collection<WebPage.Field> getFields(Job job) {
Collection<WebPage.Field> fields = new HashSet<WebPage.Field>(FIELDS);
- if (job.getConfiguration().getBoolean(PARSE_KEY, true)) {
+ if (job.getConfiguration().getBoolean(PARSE_KEY, false)) {
ParserJob parserJob = new ParserJob();
fields.addAll(parserJob.getFields(job));
}
@@ -153,7 +153,6 @@ public class FetcherJob extends NutchToo
String batchId = (String)args.get(Nutch.ARG_BATCH);
Integer threads = (Integer)args.get(Nutch.ARG_THREADS);
Boolean shouldResume = (Boolean)args.get(Nutch.ARG_RESUME);
- Boolean parse = (Boolean)args.get(Nutch.ARG_PARSE);
Integer numTasks = (Integer)args.get(Nutch.ARG_NUMTASKS);
if (threads != null && threads > 0) {
@@ -163,9 +162,6 @@ public class FetcherJob extends NutchToo
batchId = Nutch.ALL_BATCH_ID_STR;
}
getConf().set(GeneratorJob.BATCH_ID, batchId);
- if (parse != null) {
- getConf().setBoolean(PARSE_KEY, parse);
- }
if (shouldResume != null) {
getConf().setBoolean(RESUME_KEY, shouldResume);
}
@@ -200,20 +196,18 @@ public class FetcherJob extends NutchToo
* @param batchId batchId (obtained from Generator) or null to fetch all
generated fetchlists
* @param threads number of threads per map task
* @param shouldResume
- * @param parse if true, then parse content immediately, if false then a
separate
- * run of {@link ParserJob} will be needed.
* @param numTasks number of fetching tasks (reducers). If set to < 1 then
use the default,
* which is mapred.map.tasks.
* @return 0 on success
* @throws Exception
*/
- public int fetch(String batchId, int threads, boolean shouldResume, boolean
parse, int numTasks)
+ public int fetch(String batchId, int threads, boolean shouldResume, int
numTasks)
throws Exception {
LOG.info("FetcherJob: starting");
LOG.info("FetcherJob : timelimit set for : " +
getConf().getLong("fetcher.timelimit", -1));
LOG.info("FetcherJob: threads: " + getConf().getInt(THREADS_KEY, 10));
- LOG.info("FetcherJob: parsing: " + getConf().getBoolean(PARSE_KEY, true));
+ LOG.info("FetcherJob: parsing: " + getConf().getBoolean(PARSE_KEY, false));
LOG.info("FetcherJob: resuming: " + getConf().getBoolean(RESUME_KEY,
false));
if (batchId.equals(Nutch.ALL_BATCH_ID_STR)) {
LOG.info("FetcherJob: fetching all");
@@ -225,7 +219,6 @@ public class FetcherJob extends NutchToo
Nutch.ARG_BATCH, batchId,
Nutch.ARG_THREADS, threads,
Nutch.ARG_RESUME, shouldResume,
- Nutch.ARG_PARSE, parse,
Nutch.ARG_NUMTASKS, numTasks));
LOG.info("FetcherJob: done");
return 0;
@@ -268,7 +261,6 @@ public class FetcherJob extends NutchToo
public int run(String[] args) throws Exception {
int threads = -1;
boolean shouldResume = false;
- boolean parse = getConf().getBoolean(PARSE_KEY, false);
String batchId;
String usage = "Usage: FetcherJob (<batchId> | -all) [-crawlId <id>] " +
@@ -276,7 +268,6 @@ public class FetcherJob extends NutchToo
"\tbatchId\tcrawl identifier returned by Generator, or -all for all
generated batchId-s\n" +
"\t-crawlId <id>\t the id to prefix the schemas to operate on, (default:
storage.crawl.id)\n" +
"\t-threads N\tnumber of fetching threads per task\n" +
- "\t-parse\tif specified then fetcher will immediately parse fetched
content\n" +
"\t-resume\tresume interrupted job\n" +
"\t-numTasks N\tif N > 0 then use this many reduce tasks for fetching
(default: mapred.map.tasks)";
@@ -297,8 +288,6 @@ public class FetcherJob extends NutchToo
threads = Integer.parseInt(args[++i]);
} else if ("-resume".equals(args[i])) {
shouldResume = true;
- } else if ("-parse".equals(args[i])) {
- parse = true;
} else if ("-numTasks".equals(args[i])) {
numTasks = Integer.parseInt(args[++i]);
} else if ("-crawlId".equals(args[i])) {
@@ -306,7 +295,7 @@ public class FetcherJob extends NutchToo
}
}
- int fetchcode = fetch(batchId, threads, shouldResume, parse, numTasks); //
run the Fetcher
+ int fetchcode = fetch(batchId, threads, shouldResume, numTasks); // run
the Fetcher
return fetchcode;
}
Modified: nutch/branches/nutchgora/src/java/org/apache/nutch/metadata/Nutch.java
URL:
http://svn.apache.org/viewvc/nutch/branches/nutchgora/src/java/org/apache/nutch/metadata/Nutch.java?rev=1196516&r1=1196515&r2=1196516&view=diff
==============================================================================
--- nutch/branches/nutchgora/src/java/org/apache/nutch/metadata/Nutch.java
(original)
+++ nutch/branches/nutchgora/src/java/org/apache/nutch/metadata/Nutch.java Wed
Nov 2 09:26:44 2011
@@ -86,8 +86,6 @@ public interface Nutch {
public static final String ARG_RESUME = "resume";
/** Force processing even if there are locks or inconsistencies. */
public static final String ARG_FORCE = "force";
- /** Parse during fetching. */
- public static final String ARG_PARSE = "parse";
/** Sort statistics. */
public static final String ARG_SORT = "sort";
/** Solr URL. */
Modified:
nutch/branches/nutchgora/src/java/org/apache/nutch/tools/Benchmark.java
URL:
http://svn.apache.org/viewvc/nutch/branches/nutchgora/src/java/org/apache/nutch/tools/Benchmark.java?rev=1196516&r1=1196515&r2=1196516&view=diff
==============================================================================
--- nutch/branches/nutchgora/src/java/org/apache/nutch/tools/Benchmark.java
(original)
+++ nutch/branches/nutchgora/src/java/org/apache/nutch/tools/Benchmark.java Wed
Nov 2 09:26:44 2011
@@ -222,9 +222,9 @@ public class Benchmark extends Configure
LOG.info("Stopping at depth=" + i + " - no more URLs to fetch.");
break;
}
- boolean isParsing = getConf().getBoolean("fetcher.parse", true);
+ boolean isParsing = getConf().getBoolean("fetcher.parse", false);
start = System.currentTimeMillis();
- fetcher.fetch(batchId, threads, false, isParsing, -1); // fetch it
+ fetcher.fetch(batchId, threads, false, -1); // fetch it
delta = System.currentTimeMillis() - start;
res.addTiming("fetch", i + "", delta);
if (!isParsing) {
