Author: jnioche
Date: Tue Jul 20 09:10:21 2010
New Revision: 965763
URL: http://svn.apache.org/viewvc?rev=965763&view=rev
Log:
cleanup of plugin list + converted SubCollectionIndexingFilter to new API +
removed deprecated dependencies from TestRSSParser.java
Modified:
nutch/branches/nutchbase/src/plugin/build.xml
nutch/branches/nutchbase/src/plugin/parse-rss/src/test/org/apache/nutch/parse/rss/TestRSSParser.java
nutch/branches/nutchbase/src/plugin/subcollection/src/java/org/apache/nutch/indexer/subcollection/SubcollectionIndexingFilter.java
Modified: nutch/branches/nutchbase/src/plugin/build.xml
URL:
http://svn.apache.org/viewvc/nutch/branches/nutchbase/src/plugin/build.xml?rev=965763&r1=965762&r2=965763&view=diff
==============================================================================
--- nutch/branches/nutchbase/src/plugin/build.xml (original)
+++ nutch/branches/nutchbase/src/plugin/build.xml Tue Jul 20 09:10:21 2010
@@ -26,38 +26,23 @@
<!-- Build & deploy all the plugin jars. -->
<!-- ====================================================== -->
<target name="deploy">
- <!--
- <ant dir="creativecommons" target="deploy"/>
- <ant dir="feed" target="deploy"/>-->
+ <ant dir="index-anchor" target="deploy"/>
<ant dir="index-basic" target="deploy"/>
<ant dir="index-more" target="deploy"/>
- <ant dir="index-anchor" target="deploy"/>
<ant dir="languageidentifier" target="deploy"/>
<ant dir="lib-http" target="deploy"/>
<ant dir="lib-nekohtml" target="deploy"/>
<ant dir="lib-regex-filter" target="deploy"/>
<ant dir="lib-xml" target="deploy"/>
- <!--
- <ant dir="microformats-reltag" target="deploy"/>-->
<ant dir="nutch-extensionpoints" target="deploy"/>
<ant dir="protocol-ftp" target="deploy"/>
<ant dir="protocol-file" target="deploy"/>
<ant dir="protocol-http" target="deploy"/>
- <!--<ant dir="protocol-httpclient" target="deploy"/>
- <ant dir="parse-ext" target="deploy"/>-->
<ant dir="parse-js" target="deploy"/>
<ant dir="parse-rss" target="deploy"/>
- <!--<ant dir="parse-swf" target="deploy"/>
- <ant dir="parse-zip" target="deploy"/>
- -->
<ant dir="parse-tika" target="deploy"/>
<ant dir="scoring-opic" target="deploy"/>
- <!--
- <ant dir="scoring-link" target="deploy"/>
- -->
- <!--
<ant dir="subcollection" target="deploy"/>
- <ant dir="tld" target="deploy"/>-->
<ant dir="urlfilter-automaton" target="deploy"/>
<ant dir="urlfilter-domain" target="deploy" />
<ant dir="urlfilter-prefix" target="deploy"/>
@@ -67,6 +52,17 @@
<ant dir="urlnormalizer-basic" target="deploy"/>
<ant dir="urlnormalizer-pass" target="deploy"/>
<ant dir="urlnormalizer-regex" target="deploy"/>
+ <!--
+ <ant dir="creativecommons" target="deploy"/>
+ <ant dir="feed" target="deploy"/>
+ <ant dir="microformats-reltag" target="deploy"/>
+ <ant dir="parse-ext" target="deploy"/>
+ <ant dir="parse-swf" target="deploy"/>
+ <ant dir="parse-zip" target="deploy"/>
+ <ant dir="protocol-httpclient" target="deploy"/>
+ <ant dir="scoring-link" target="deploy"/>
+ <ant dir="tld" target="deploy"/>
+ -->
</target>
<!-- ====================================================== -->
@@ -74,6 +70,7 @@
<!-- ====================================================== -->
<target name="test">
<parallel threadCount="2">
+ <ant dir="parse-rss" target="test"/>
<ant dir="parse-tika" target="test"/>
<ant dir="protocol-file" target="test"/>
<ant dir="parse-rss" target="test"/>
@@ -88,15 +85,14 @@
<ant dir="urlnormalizer-regex" target="test"/>
<ant dir="languageidentifier" target="test"/>
<ant dir="lib-http" target="test"/>
+ <ant dir="subcollection" target="test"/>
<!--
- <ant dir="protocol-httpclient" target="test"/>
<ant dir="creativecommons" target="test"/>
- <ant dir="parse-ext" target="test"/>
<ant dir="feed" target="test"/>
+ <ant dir="parse-ext" target="test"/>
<ant dir="parse-swf" target="test"/>
<ant dir="parse-zip" target="test"/>
- <ant dir="query-url" target="test"/>
- <ant dir="subcollection" target="test"/>
+ <ant dir="protocol-httpclient" target="test"/>
-->
</parallel>
</target>
@@ -107,30 +103,28 @@
<target name="clean">
<ant dir="creativecommons" target="clean"/>
<ant dir="feed" target="clean"/>
- <ant dir="index-basic" target="clean"/>
<ant dir="index-anchor" target="clean"/>
+ <ant dir="index-basic" target="clean"/>
<ant dir="index-more" target="clean"/>
<ant dir="languageidentifier" target="clean"/>
- <ant dir="lib-commons-httpclient" target="clean"/>
<ant dir="lib-http" target="clean"/>
- <ant dir="lib-lucene-analyzers" target="clean"/>
<ant dir="lib-nekohtml" target="clean"/>
<ant dir="lib-regex-filter" target="clean"/>
<ant dir="lib-xml" target="clean"/>
<ant dir="microformats-reltag" target="clean"/>
<ant dir="nutch-extensionpoints" target="clean"/>
- <ant dir="protocol-file" target="clean"/>
- <ant dir="protocol-ftp" target="clean"/>
- <ant dir="protocol-http" target="clean"/>
- <ant dir="protocol-httpclient" target="clean"/>
<ant dir="parse-ext" target="clean"/>
<ant dir="parse-js" target="clean"/>
<ant dir="parse-rss" target="clean"/>
<ant dir="parse-swf" target="clean"/>
<ant dir="parse-tika" target="clean"/>
<ant dir="parse-zip" target="clean"/>
- <ant dir="scoring-opic" target="clean"/>
+ <ant dir="protocol-file" target="clean"/>
+ <ant dir="protocol-ftp" target="clean"/>
+ <ant dir="protocol-httpclient" target="clean"/>
+ <ant dir="protocol-http" target="clean"/>
<ant dir="scoring-link" target="clean"/>
+ <ant dir="scoring-opic" target="clean"/>
<ant dir="subcollection" target="clean"/>
<ant dir="tld" target="clean"/>
<ant dir="urlfilter-automaton" target="clean"/>
Modified:
nutch/branches/nutchbase/src/plugin/parse-rss/src/test/org/apache/nutch/parse/rss/TestRSSParser.java
URL:
http://svn.apache.org/viewvc/nutch/branches/nutchbase/src/plugin/parse-rss/src/test/org/apache/nutch/parse/rss/TestRSSParser.java?rev=965763&r1=965762&r2=965763&view=diff
==============================================================================
---
nutch/branches/nutchbase/src/plugin/parse-rss/src/test/org/apache/nutch/parse/rss/TestRSSParser.java
(original)
+++
nutch/branches/nutchbase/src/plugin/parse-rss/src/test/org/apache/nutch/parse/rss/TestRSSParser.java
Tue Jul 20 09:10:21 2010
@@ -36,7 +36,6 @@ import org.apache.nutch.util.MimeUtil;
import org.apache.nutch.util.NutchConfiguration;
import org.apache.hadoop.io.Text;
-import org.apache.nutch.crawl.CrawlDatum;
import org.apache.tika.mime.MimeType;
import junit.framework.TestCase;
Modified:
nutch/branches/nutchbase/src/plugin/subcollection/src/java/org/apache/nutch/indexer/subcollection/SubcollectionIndexingFilter.java
URL:
http://svn.apache.org/viewvc/nutch/branches/nutchbase/src/plugin/subcollection/src/java/org/apache/nutch/indexer/subcollection/SubcollectionIndexingFilter.java?rev=965763&r1=965762&r2=965763&view=diff
==============================================================================
---
nutch/branches/nutchbase/src/plugin/subcollection/src/java/org/apache/nutch/indexer/subcollection/SubcollectionIndexingFilter.java
(original)
+++
nutch/branches/nutchbase/src/plugin/subcollection/src/java/org/apache/nutch/indexer/subcollection/SubcollectionIndexingFilter.java
Tue Jul 20 09:10:21 2010
@@ -16,59 +16,64 @@
*/
package org.apache.nutch.indexer.subcollection;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.Configured;
-import org.apache.hadoop.io.Text;
+import java.util.ArrayList;
+import java.util.Collection;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-
-import org.apache.nutch.parse.Parse;
-import org.apache.nutch.util.NutchConfiguration;
-
-import org.apache.nutch.indexer.IndexingFilter;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
+import org.apache.nutch.collection.CollectionManager;
import org.apache.nutch.indexer.IndexingException;
+import org.apache.nutch.indexer.IndexingFilter;
import org.apache.nutch.indexer.NutchDocument;
+import org.apache.nutch.storage.WebPage;
+import org.apache.nutch.storage.WebPage.Field;
+import org.apache.nutch.util.NutchConfiguration;
-import org.apache.nutch.collection.CollectionManager;
-import org.apache.nutch.crawl.CrawlDatum;
-import org.apache.nutch.crawl.Inlinks;
-
-
-public class SubcollectionIndexingFilter extends Configured implements
IndexingFilter {
+public class SubcollectionIndexingFilter extends Configured implements
+ IndexingFilter {
- public SubcollectionIndexingFilter(){
- super(NutchConfiguration.create());
- }
-
- public SubcollectionIndexingFilter(Configuration conf) {
- super(conf);
- }
-
- /**
- * Doc field name
- */
- public static final String FIELD_NAME = "subcollection";
-
- /**
- * Logger
- */
- public static final Log LOG =
LogFactory.getLog(SubcollectionIndexingFilter.class);
-
- /**
- * "Mark" document to be a part of subcollection
- *
- * @param doc
- * @param url
- */
- private void addSubCollectionField(NutchDocument doc, String url) {
- String collname =
CollectionManager.getCollectionManager(getConf()).getSubCollections(url);
- doc.add(FIELD_NAME, collname);
- }
-
- public NutchDocument filter(NutchDocument doc, Parse parse, Text url,
CrawlDatum datum, Inlinks inlinks) throws IndexingException {
- String sUrl = url.toString();
- addSubCollectionField(doc, sUrl);
- return doc;
- }
+ public SubcollectionIndexingFilter() {
+ super(NutchConfiguration.create());
+ }
+
+ public SubcollectionIndexingFilter(Configuration conf) {
+ super(conf);
+ }
+
+ /**
+ * Doc field name
+ */
+ public static final String FIELD_NAME = "subcollection";
+
+ /**
+ * Logger
+ */
+ public static final Log LOG = LogFactory
+ .getLog(SubcollectionIndexingFilter.class);
+
+ /**
+ * "Mark" document to be a part of subcollection
+ *
+ * @param doc
+ * @param url
+ */
+ private void addSubCollectionField(NutchDocument doc, String url) {
+ String collname =
CollectionManager.getCollectionManager(getConf())
+ .getSubCollections(url);
+ doc.add(FIELD_NAME, collname);
+ }
+
+ @Override
+ public Collection<Field> getFields() {
+ return new ArrayList<Field>();
+ }
+
+ @Override
+ public NutchDocument filter(NutchDocument doc, String url, WebPage page)
+ throws IndexingException {
+ addSubCollectionField(doc, url);
+ return doc;
+ }
}
