Author: jnioche
Date: Mon Mar 29 12:12:09 2010
New Revision: 928746

URL: http://svn.apache.org/viewvc?rev=928746&view=rev
Log:
NUTCH-784 : CrawlDBScanner

Added:
    lucene/nutch/trunk/src/java/org/apache/nutch/tools/CrawlDBScanner.java
Modified:
    lucene/nutch/trunk/CHANGES.txt

Modified: lucene/nutch/trunk/CHANGES.txt
URL: 
http://svn.apache.org/viewvc/lucene/nutch/trunk/CHANGES.txt?rev=928746&r1=928745&r2=928746&view=diff
==============================================================================
--- lucene/nutch/trunk/CHANGES.txt (original)
+++ lucene/nutch/trunk/CHANGES.txt Mon Mar 29 12:12:09 2010
@@ -2,6 +2,8 @@ Nutch Change Log
 
 Unreleased Changes
 
+* NUTCH-784 CrawlDBScanner (jnioche)
+
 * NUTCH-762 Generator can generate several segments in one parse of the 
crawlDB (jnioche)
 
 * NUTCH-740 Configuration option to override default language for fetched 
pages (Marcin Okraszewski via jnioche)

Added: lucene/nutch/trunk/src/java/org/apache/nutch/tools/CrawlDBScanner.java
URL: 
http://svn.apache.org/viewvc/lucene/nutch/trunk/src/java/org/apache/nutch/tools/CrawlDBScanner.java?rev=928746&view=auto
==============================================================================
--- lucene/nutch/trunk/src/java/org/apache/nutch/tools/CrawlDBScanner.java 
(added)
+++ lucene/nutch/trunk/src/java/org/apache/nutch/tools/CrawlDBScanner.java Mon 
Mar 29 12:12:09 2010
@@ -0,0 +1,165 @@
+package org.apache.nutch.tools;
+
+import java.io.IOException;
+import java.util.Iterator;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapred.FileInputFormat;
+import org.apache.hadoop.mapred.FileOutputFormat;
+import org.apache.hadoop.mapred.JobClient;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.MapFileOutputFormat;
+import org.apache.hadoop.mapred.Mapper;
+import org.apache.hadoop.mapred.OutputCollector;
+import org.apache.hadoop.mapred.Reducer;
+import org.apache.hadoop.mapred.Reporter;
+import org.apache.hadoop.mapred.SequenceFileInputFormat;
+import org.apache.hadoop.mapred.TextOutputFormat;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+import org.apache.nutch.crawl.CrawlDatum;
+import org.apache.nutch.crawl.CrawlDb;
+import org.apache.nutch.util.NutchConfiguration;
+import org.apache.nutch.util.NutchJob;
+
+/**
+ * Dumps all the entries matching a regular expression on their URL. Generates 
a
+ * text representation of the CrawlDatum-s or binary objects which can then be
+ * used as a new CrawlDB. The dump mechanism of the crawldb reader is not very
+ * useful on large crawldbs as the ouput can be extremely large and the -url
+ * function can't help if we don't know what url we want to have a look at.
+ * 
+ * @author : Julien Nioche
+ */
+
+public class CrawlDBScanner extends Configured implements Tool,
+    Mapper<Text,CrawlDatum,Text,CrawlDatum>, 
Reducer<Text,CrawlDatum,Text,CrawlDatum> {
+
+  public static final Log LOG = LogFactory.getLog(CrawlDBScanner.class);
+
+  public CrawlDBScanner() {}
+
+  public CrawlDBScanner(Configuration conf) {
+    setConf(conf);
+  }
+
+  public void close() {}
+
+  private String regex = null;
+  private String status = null;
+
+  public void configure(JobConf job) {
+    regex = job.get("CrawlDBScanner.regex");
+    status = job.get("CrawlDBScanner.status");
+  }
+
+  public void map(Text url, CrawlDatum crawlDatum,
+      OutputCollector<Text,CrawlDatum> output, Reporter reporter) throws 
IOException {
+
+    // check status
+    if (status != null
+        && 
!status.equalsIgnoreCase(CrawlDatum.getStatusName(crawlDatum.getStatus()))) 
return;
+
+    // if URL matched regexp dump it
+    if (url.toString().matches(regex)) {
+      output.collect(url, crawlDatum);
+    }
+  }
+
+  public void reduce(Text key, Iterator<CrawlDatum> values,
+      OutputCollector<Text,CrawlDatum> output, Reporter reporter) throws 
IOException {
+    while (values.hasNext()) {
+      CrawlDatum val = values.next();
+      output.collect(key, val);
+    }
+  }
+
+  private void scan(Path crawlDb, Path outputPath, String regex, String status,
+      boolean text) throws IOException {
+
+    JobConf job = new NutchJob(getConf());
+
+    job.setJobName("Scan : " + crawlDb + " for URLS matching : " + regex);
+
+    job.set("CrawlDBScanner.regex", regex);
+    if (status != null) job.set("CrawlDBScanner.status", status);
+
+    FileInputFormat.addInputPath(job, new Path(crawlDb, CrawlDb.CURRENT_NAME));
+    job.setInputFormat(SequenceFileInputFormat.class);
+
+    job.setMapperClass(CrawlDBScanner.class);
+    job.setReducerClass(CrawlDBScanner.class);
+
+    FileOutputFormat.setOutputPath(job, outputPath);
+
+    // if we want a text dump of the entries
+    // in order to check something - better to use the text format and avoid
+    // compression
+    if (text) {
+      job.set("mapred.output.compress", "false");
+      job.setOutputFormat(TextOutputFormat.class);
+    }
+    // otherwise what we will actually create is a mini-crawlDB which can be
+    // then used
+    // for debugging
+    else {
+      job.setOutputFormat(MapFileOutputFormat.class);
+    }
+
+    job.setMapOutputKeyClass(Text.class);
+    job.setMapOutputValueClass(CrawlDatum.class);
+
+    job.setOutputKeyClass(Text.class);
+    job.setOutputValueClass(CrawlDatum.class);
+
+    try {
+      JobClient.runJob(job);
+    } catch (IOException e) {
+      throw e;
+    }
+  }
+
+  public static void main(String args[]) throws Exception {
+    int res = ToolRunner.run(NutchConfiguration.create(), new 
CrawlDBScanner(), args);
+    System.exit(res);
+  }
+
+  public int run(String[] args) throws Exception {
+    if (args.length < 3) {
+      System.err
+          .println("Usage: CrawlDBScanner <crawldb> <output> <regex> [-s 
<status>] <-text>");
+      return -1;
+    }
+
+    boolean text = false;
+
+    Path dbDir = new Path(args[0]);
+    Path output = new Path(args[1]);
+
+    String status = null;
+
+    for (int i = 2; i < args.length; i++) {
+      if (args[i].equals("-text")) {
+        text = true;
+      } else if (args[i].equals("-s")) {
+        i++;
+        status = args[i];
+      }
+    }
+
+    try {
+      scan(dbDir, output, args[2], status, text);
+      return 0;
+    } catch (Exception e) {
+      LOG.fatal("CrawlDBScanner: " + StringUtils.stringifyException(e));
+      return -1;
+    }
+  }
+
+}


Reply via email to