Author: fenglu
Date: Thu Sep 5 14:40:25 2013
New Revision: 1520332
URL: http://svn.apache.org/r1520332
Log:
NUTCH-1556 enabling updatedb to accept batchId
Modified:
nutch/branches/2.x/CHANGES.txt
nutch/branches/2.x/src/bin/crawl
nutch/branches/2.x/src/java/org/apache/nutch/crawl/DbUpdateMapper.java
nutch/branches/2.x/src/java/org/apache/nutch/crawl/DbUpdaterJob.java
Modified: nutch/branches/2.x/CHANGES.txt
URL:
http://svn.apache.org/viewvc/nutch/branches/2.x/CHANGES.txt?rev=1520332&r1=1520331&r2=1520332&view=diff
==============================================================================
--- nutch/branches/2.x/CHANGES.txt (original)
+++ nutch/branches/2.x/CHANGES.txt Thu Sep 5 14:40:25 2013
@@ -2,6 +2,8 @@ Nutch Change Log
Current Development
+* NUTCH-1556 enabling updatedb to accept batchId (kaveh minooie,Feng)
+
* NUTCH-1619 Writes Dmoz Description and Title information to db with snippet
argument ( Yasin Kılınç via feng)
* NUTCH-1631 Display Document Count Added To Solr Server (Furkan KAMACI via
lewismc)
Modified: nutch/branches/2.x/src/bin/crawl
URL:
http://svn.apache.org/viewvc/nutch/branches/2.x/src/bin/crawl?rev=1520332&r1=1520331&r2=1520332&view=diff
==============================================================================
--- nutch/branches/2.x/src/bin/crawl (original)
+++ nutch/branches/2.x/src/bin/crawl Thu Sep 5 14:40:25 2013
@@ -141,7 +141,7 @@ do
# updatedb with this batch
echo "CrawlDB update for $CRAWL_ID"
- $bin/nutch updatedb $commonOptions -crawlId $CRAWL_ID
+ $bin/nutch updatedb $commonOptions -batchId $batchId -crawlId $CRAWL_ID
if [ $? -ne 0 ]
then exit $?
Modified: nutch/branches/2.x/src/java/org/apache/nutch/crawl/DbUpdateMapper.java
URL:
http://svn.apache.org/viewvc/nutch/branches/2.x/src/java/org/apache/nutch/crawl/DbUpdateMapper.java?rev=1520332&r1=1520331&r2=1520332&view=diff
==============================================================================
--- nutch/branches/2.x/src/java/org/apache/nutch/crawl/DbUpdateMapper.java
(original)
+++ nutch/branches/2.x/src/java/org/apache/nutch/crawl/DbUpdateMapper.java Thu
Sep 5 14:40:25 2013
@@ -23,6 +23,9 @@ import java.util.Map;
import java.util.Map.Entry;
import org.apache.avro.util.Utf8;
+import org.apache.nutch.metadata.Nutch;
+import org.apache.nutch.storage.Mark;
+import org.apache.nutch.util.NutchJob;
import org.slf4j.Logger;
import org.apache.hadoop.io.FloatWritable;
import org.apache.hadoop.util.StringUtils;
@@ -41,6 +44,8 @@ extends GoraMapper<String, WebPage, UrlW
private ScoringFilters scoringFilters;
private final List<ScoreDatum> scoreData = new ArrayList<ScoreDatum>();
+
+ private Utf8 batchId;
//reuse writables
private UrlWithScore urlWithScore = new UrlWithScore();
@@ -51,6 +56,14 @@ extends GoraMapper<String, WebPage, UrlW
public void map(String key, WebPage page, Context context)
throws IOException, InterruptedException {
+ Utf8 mark = Mark.GENERATE_MARK.checkMark(page);
+ if(!NutchJob.shouldProcess(mark,batchId)) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Skipping " + TableUtil.unreverseUrl(key) + "; different
batch id (" + mark + ")");
+ }
+ return;
+ }
+
String url = TableUtil.unreverseUrl(key);
scoreData.clear();
@@ -93,6 +106,7 @@ extends GoraMapper<String, WebPage, UrlW
public void setup(Context context) {
scoringFilters = new ScoringFilters(context.getConfiguration());
pageWritable = new WebPageWritable(context.getConfiguration(), null);
+ batchId = new
Utf8(context.getConfiguration().get(Nutch.BATCH_NAME_KEY,Nutch.ALL_BATCH_ID_STR));
}
}
Modified: nutch/branches/2.x/src/java/org/apache/nutch/crawl/DbUpdaterJob.java
URL:
http://svn.apache.org/viewvc/nutch/branches/2.x/src/java/org/apache/nutch/crawl/DbUpdaterJob.java?rev=1520332&r1=1520331&r2=1520332&view=diff
==============================================================================
--- nutch/branches/2.x/src/java/org/apache/nutch/crawl/DbUpdaterJob.java
(original)
+++ nutch/branches/2.x/src/java/org/apache/nutch/crawl/DbUpdaterJob.java Thu
Sep 5 14:40:25 2013
@@ -73,12 +73,17 @@ public class DbUpdaterJob extends NutchT
public Map<String,Object> run(Map<String,Object> args) throws Exception {
String crawlId = (String)args.get(Nutch.ARG_CRAWL);
+ String batchId = (String)args.get(Nutch.ARG_BATCH);
numJobs = 1;
currentJobNum = 0;
currentJob = new NutchJob(getConf(), "update-table");
if (crawlId != null) {
currentJob.getConfiguration().set(Nutch.CRAWL_ID_KEY, crawlId);
}
+ if (batchId == null) {
+ batchId = Nutch.ALL_BATCH_ID_STR;
+ }
+ getConf().set(Nutch.BATCH_NAME_KEY, batchId);
//job.setBoolean(ALL, updateAll);
ScoringFilters scoringFilters = new ScoringFilters(getConf());
HashSet<WebPage.Field> fields = new HashSet<WebPage.Field>(FIELDS);
@@ -100,23 +105,46 @@ public class DbUpdaterJob extends NutchT
return results;
}
- private int updateTable(String crawlId) throws Exception {
+ private int updateTable(String crawlId,String batchId) throws Exception {
LOG.info("DbUpdaterJob: starting");
- run(ToolUtil.toArgMap(Nutch.ARG_CRAWL, crawlId));
+ if (batchId.equals(Nutch.ALL_BATCH_ID_STR)) {
+ LOG.info("DbUpdaterJob: updatinging all");
+ } else {
+ LOG.info("DbUpdaterJob: batchId: " + batchId);
+ }
+ run(ToolUtil.toArgMap(Nutch.ARG_CRAWL, crawlId,
+ Nutch.ARG_BATCH, batchId));
LOG.info("DbUpdaterJob: done");
return 0;
}
public int run(String[] args) throws Exception {
String crawlId = null;
+ String batchId;
+
+ String usage = "Usage: DbUpdaterJob (<batchId> | -all) [-crawlId <id>] " +
+ " <batchId> - crawl identifier returned by Generator, or
-all for all \n \t \t generated batchId-s\n" +
+ " -crawlId <id> - the id to prefix the schemas to operate on,
\n \t \t (default: storage.crawl.id)\n";
+
if (args.length == 0) {
- //
- } else if (args.length == 2 && "-crawlId".equals(args[0])) {
- crawlId = args[1];
- } else {
- throw new IllegalArgumentException("usage: " + "(-crawlId <id>)");
+ System.err.println(usage);
+ return -1;
+ }
+
+ batchId = args[0];
+ if (!batchId.equals("-all") && batchId.startsWith("-")) {
+ System.err.println(usage);
+ return -1;
+ }
+
+ for (int i = 1; i < args.length; i++) {
+ if ("-crawlId".equals(args[i])) {
+ getConf().set(Nutch.CRAWL_ID_KEY, args[++i]);
+ } else {
+ throw new IllegalArgumentException("arg " +args[i]+ " not recognized");
+ }
}
- return updateTable(crawlId);
+ return updateTable(crawlId,batchId);
}
public static void main(String[] args) throws Exception {
