Author: jnioche
Date: Tue Jul 20 11:18:18 2010
New Revision: 965806
URL: http://svn.apache.org/viewvc?rev=965806&view=rev
Log:
ported microformat plugin
Modified:
nutch/branches/nutchbase/src/plugin/build.xml
nutch/branches/nutchbase/src/plugin/microformats-reltag/src/java/org/apache/nutch/microformats/reltag/RelTagIndexingFilter.java
nutch/branches/nutchbase/src/plugin/microformats-reltag/src/java/org/apache/nutch/microformats/reltag/RelTagParser.java
Modified: nutch/branches/nutchbase/src/plugin/build.xml
URL:
http://svn.apache.org/viewvc/nutch/branches/nutchbase/src/plugin/build.xml?rev=965806&r1=965805&r2=965806&view=diff
==============================================================================
--- nutch/branches/nutchbase/src/plugin/build.xml (original)
+++ nutch/branches/nutchbase/src/plugin/build.xml Tue Jul 20 11:18:18 2010
@@ -35,6 +35,7 @@
<ant dir="lib-nekohtml" target="deploy"/>
<ant dir="lib-regex-filter" target="deploy"/>
<ant dir="lib-xml" target="deploy"/>
+ <ant dir="microformats-reltag" target="deploy"/>
<ant dir="nutch-extensionpoints" target="deploy"/>
<ant dir="protocol-ftp" target="deploy"/>
<ant dir="protocol-file" target="deploy"/>
@@ -57,7 +58,6 @@
<ant dir="urlnormalizer-regex" target="deploy"/>
<!--
<ant dir="feed" target="deploy"/>
- <ant dir="microformats-reltag" target="deploy"/>
<ant dir="parse-ext" target="deploy"/>
<ant dir="parse-swf" target="deploy"/>
<ant dir="parse-zip" target="deploy"/>
Modified:
nutch/branches/nutchbase/src/plugin/microformats-reltag/src/java/org/apache/nutch/microformats/reltag/RelTagIndexingFilter.java
URL:
http://svn.apache.org/viewvc/nutch/branches/nutchbase/src/plugin/microformats-reltag/src/java/org/apache/nutch/microformats/reltag/RelTagIndexingFilter.java?rev=965806&r1=965805&r2=965806&view=diff
==============================================================================
---
nutch/branches/nutchbase/src/plugin/microformats-reltag/src/java/org/apache/nutch/microformats/reltag/RelTagIndexingFilter.java
(original)
+++
nutch/branches/nutchbase/src/plugin/microformats-reltag/src/java/org/apache/nutch/microformats/reltag/RelTagIndexingFilter.java
Tue Jul 20 11:18:18 2010
@@ -16,63 +16,64 @@
*/
package org.apache.nutch.microformats.reltag;
-
// Nutch imports
-import org.apache.nutch.crawl.CrawlDatum;
-import org.apache.nutch.crawl.Inlinks;
-import org.apache.nutch.indexer.IndexingFilter;
-import org.apache.nutch.indexer.IndexingException;
-import org.apache.nutch.indexer.NutchDocument;
-import org.apache.hadoop.io.Text;
-import org.apache.nutch.parse.Parse;
+import java.nio.ByteBuffer;
+import java.util.Collection;
+import java.util.HashSet;
-// Hadoop imports
+import org.apache.avro.util.Utf8;
import org.apache.hadoop.conf.Configuration;
-
+import org.apache.nutch.indexer.IndexingException;
+import org.apache.nutch.indexer.IndexingFilter;
+import org.apache.nutch.indexer.NutchDocument;
+import org.apache.nutch.storage.WebPage;
+import org.apache.nutch.storage.WebPage.Field;
/**
- * An {...@link org.apache.nutch.indexer.IndexingFilter} that
- * add <code>tag</code> field(s) to the document.
- *
+ * An {...@link org.apache.nutch.indexer.IndexingFilter} that add
<code>tag</code>
+ * field(s) to the document.
+ *
* @see <a href="http://www.microformats.org/wiki/rel-tag";>
* http://www.microformats.org/wiki/rel-tag</a>
* @author Jérôme Charron
*/
public class RelTagIndexingFilter implements IndexingFilter {
-
- private Configuration conf;
+ private Configuration conf;
+
+ private static final Collection<WebPage.Field> FIELDS = new
HashSet<WebPage.Field>();
+ static {
+ FIELDS.add(WebPage.Field.BASE_URL);
+ FIELDS.add(WebPage.Field.METADATA);
+ }
+
+ @Override
+ public Collection<Field> getFields() {
+ return FIELDS;
+ }
+
+ public void setConf(Configuration conf) {
+ this.conf = conf;
+ }
+
+ public Configuration getConf() {
+ return this.conf;
+ }
+
+ @Override
+ public NutchDocument filter(NutchDocument doc, String url, WebPage page)
+ throws IndexingException {
+ // Check if some Rel-Tags found, possibly put there by
RelTagParser
+ ByteBuffer bb = page.getFromMetadata(new
Utf8(RelTagParser.REL_TAG));
+
+ if (bb != null) {
+ String[] tags = new String(bb.array()).split("\t");
+ for (int i = 0; i < tags.length; i++) {
+ doc.add("tag", tags[i]);
+ }
+ }
- // Inherited JavaDoc
- public NutchDocument filter(NutchDocument doc, Parse parse, Text url,
CrawlDatum datum, Inlinks inlinks)
- throws IndexingException {
-
- // Check if some Rel-Tags found, possibly put there by RelTagParser
- String[] tags =
parse.getData().getParseMeta().getValues(RelTagParser.REL_TAG);
- if (tags != null) {
- for (int i=0; i<tags.length; i++) {
- doc.add("tag", tags[i]);
- }
- }
-
- return doc;
- }
-
- /* ----------------------------- *
- * <implementation:Configurable> *
- * ----------------------------- */
-
- public void setConf(Configuration conf) {
- this.conf = conf;
- }
-
- public Configuration getConf() {
- return this.conf;
- }
-
- /* ------------------------------ *
- * </implementation:Configurable> *
- * ------------------------------ */
-
+ return doc;
+ }
}
Modified:
nutch/branches/nutchbase/src/plugin/microformats-reltag/src/java/org/apache/nutch/microformats/reltag/RelTagParser.java
URL:
http://svn.apache.org/viewvc/nutch/branches/nutchbase/src/plugin/microformats-reltag/src/java/org/apache/nutch/microformats/reltag/RelTagParser.java?rev=965806&r1=965805&r2=965806&view=diff
==============================================================================
---
nutch/branches/nutchbase/src/plugin/microformats-reltag/src/java/org/apache/nutch/microformats/reltag/RelTagParser.java
(original)
+++
nutch/branches/nutchbase/src/plugin/microformats-reltag/src/java/org/apache/nutch/microformats/reltag/RelTagParser.java
Tue Jul 20 11:18:18 2010
@@ -19,142 +19,141 @@ package org.apache.nutch.microformats.re
// JDK imports
import java.net.URL;
import java.net.URLDecoder;
+import java.nio.ByteBuffer;
+import java.util.Collection;
+import java.util.HashSet;
import java.util.Iterator;
import java.util.Set;
import java.util.TreeSet;
-import org.w3c.dom.DocumentFragment;
-import org.w3c.dom.NamedNodeMap;
-import org.w3c.dom.Node;
-import org.w3c.dom.NodeList;
-// Commons Logging imports
+import org.apache.avro.util.Utf8;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-
-// Nutch imports
-import org.apache.nutch.metadata.Metadata;
+import org.apache.hadoop.conf.Configuration;
import org.apache.nutch.parse.HTMLMetaTags;
-import org.apache.nutch.parse.Parse;
-import org.apache.nutch.parse.ParseResult;
import org.apache.nutch.parse.HtmlParseFilter;
-import org.apache.nutch.protocol.Content;
+import org.apache.nutch.parse.Parse;
+import org.apache.nutch.storage.WebPage;
+import org.apache.nutch.storage.WebPage.Field;
import org.apache.nutch.util.StringUtil;
-
-// Hadoop imports
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.io.Text;
-
+import org.w3c.dom.DocumentFragment;
+import org.w3c.dom.NamedNodeMap;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
/**
* Adds microformat rel-tags of document if found.
- *
+ *
* @see <a href="http://www.microformats.org/wiki/rel-tag";>
* http://www.microformats.org/wiki/rel-tag</a>
* @author Jérôme Charron
*/
public class RelTagParser implements HtmlParseFilter {
-
- public final static Log LOG = LogFactory.getLog(RelTagParser.class);
- public final static String REL_TAG = "Rel-Tag";
-
-
- private Configuration conf = null;
-
-
- /**
- * Scan the HTML document looking at possible rel-tags
- */
- public ParseResult filter(Content content, ParseResult parseResult,
- HTMLMetaTags metaTags, DocumentFragment doc) {
-
- // get parse obj
- Parse parse = parseResult.get(content.getUrl());
- // Trying to find the document's rel-tags
- Parser parser = new Parser(doc);
- Set tags = parser.getRelTags();
- Iterator iter = tags.iterator();
- Metadata metadata = parse.getData().getParseMeta();
- while (iter.hasNext()) {
- metadata.add(REL_TAG, (String) iter.next());
- }
- return parseResult;
- }
-
- private static class Parser {
-
- Set tags = null;
-
- Parser(Node node) {
- tags = new TreeSet();
- parse(node);
- }
-
- Set getRelTags() {
- return tags;
- }
-
- void parse(Node node) {
-
- if (node.getNodeType() == Node.ELEMENT_NODE) {
- // Look for <a> tag
- if ("a".equalsIgnoreCase(node.getNodeName())) {
- NamedNodeMap attrs = node.getAttributes();
- Node hrefNode = attrs.getNamedItem("href");
- // Checks that it contains a href attribute
- if (hrefNode != null) {
- Node relNode = attrs.getNamedItem("rel");
- // Checks that it contains a rel attribute too
- if (relNode != null) {
- // Finaly checks that rel=tag
- if ("tag".equalsIgnoreCase(relNode.getNodeValue())) {
- String tag = parseTag(hrefNode.getNodeValue());
- if (!StringUtil.isEmpty(tag)) {
- tags.add(tag);
- }
- }
- }
- }
- }
- }
-
- // Recurse
- NodeList children = node.getChildNodes();
- for (int i=0; children != null && i<children.getLength(); i++) {
- parse(children.item(i));
- }
- }
-
- private final static String parseTag(String url) {
- String tag = null;
- try {
- URL u = new URL(url);
- String path = u.getPath();
- tag = URLDecoder.decode(path.substring(path.lastIndexOf('/') + 1),
"UTF-8");
- } catch (Exception e) {
- // Malformed tag...
- tag = null;
- }
- return tag;
- }
-
- }
-
-
- /* ----------------------------- *
- * <implementation:Configurable> *
- * ----------------------------- */
-
- public void setConf(Configuration conf) {
- this.conf = conf;
- }
-
- public Configuration getConf() {
- return this.conf;
- }
-
- /* ------------------------------ *
- * </implementation:Configurable> *
- * ------------------------------ */
-
+ public final static Log LOG = LogFactory.getLog(RelTagParser.class);
+
+ public final static String REL_TAG = "Rel-Tag";
+
+ private Configuration conf = null;
+
+ private static class Parser {
+
+ Set<String> tags = null;
+
+ Parser(Node node) {
+ tags = new TreeSet<String>();
+ parse(node);
+ }
+
+ Set<String> getRelTags() {
+ return tags;
+ }
+
+ void parse(Node node) {
+
+ if (node.getNodeType() == Node.ELEMENT_NODE) {
+ // Look for <a> tag
+ if ("a".equalsIgnoreCase(node.getNodeName())) {
+ NamedNodeMap attrs =
node.getAttributes();
+ Node hrefNode =
attrs.getNamedItem("href");
+ // Checks that it contains a href
attribute
+ if (hrefNode != null) {
+ Node relNode =
attrs.getNamedItem("rel");
+ // Checks that it contains a
rel attribute too
+ if (relNode != null) {
+ // Finaly checks that
rel=tag
+ if
("tag".equalsIgnoreCase(relNode.getNodeValue())) {
+ String tag =
parseTag(hrefNode.getNodeValue());
+ if
(!StringUtil.isEmpty(tag)) {
+
tags.add(tag);
+ }
+ }
+ }
+ }
+ }
+ }
+
+ // Recurse
+ NodeList children = node.getChildNodes();
+ for (int i = 0; children != null && i <
children.getLength(); i++) {
+ parse(children.item(i));
+ }
+ }
+
+ private final static String parseTag(String url) {
+ String tag = null;
+ try {
+ URL u = new URL(url);
+ String path = u.getPath();
+ tag = URLDecoder.decode(
+
path.substring(path.lastIndexOf('/') + 1), "UTF-8");
+ } catch (Exception e) {
+ // Malformed tag...
+ tag = null;
+ }
+ return tag;
+ }
+
+ }
+
+ public void setConf(Configuration conf) {
+ this.conf = conf;
+ }
+
+ public Configuration getConf() {
+ return this.conf;
+ }
+
+ private static final Collection<WebPage.Field> FIELDS = new
HashSet<WebPage.Field>();
+
+ static {
+ FIELDS.add(WebPage.Field.BASE_URL);
+ FIELDS.add(WebPage.Field.METADATA);
+ }
+
+ @Override
+ public Collection<Field> getFields() {
+ return FIELDS;
+ }
+
+ @Override
+ /**
+ * Scan the HTML document looking at possible rel-tags
+ */
+ public Parse filter(String url, WebPage page, Parse parse,
+ HTMLMetaTags metaTags, DocumentFragment doc) {
+ // Trying to find the document's rel-tags
+ Parser parser = new Parser(doc);
+ Set<String> tags = parser.getRelTags();
+ // can't store multiple values in page metadata -> separate by
tabs
+ StringBuffer sb = new StringBuffer();
+ Iterator<String> iter = tags.iterator();
+ while (iter.hasNext()) {
+ sb.append(iter.next());
+ sb.append("\t");
+ }
+ ByteBuffer bb = ByteBuffer.wrap(sb.toString().getBytes());
+ page.putToMetadata(new Utf8(REL_TAG), bb);
+ return parse;
+ }
}
