Author: jnioche
Date: Tue Jul 20 09:22:55 2010
New Revision: 965768
URL: http://svn.apache.org/viewvc?rev=965768&view=rev
Log:
Ported tld plugin
Modified:
nutch/branches/nutchbase/src/plugin/build.xml
nutch/branches/nutchbase/src/plugin/tld/src/java/org/apache/nutch/indexer/tld/TLDIndexingFilter.java
nutch/branches/nutchbase/src/plugin/tld/src/java/org/apache/nutch/scoring/tld/TLDScoringFilter.java
Modified: nutch/branches/nutchbase/src/plugin/build.xml
URL:
http://svn.apache.org/viewvc/nutch/branches/nutchbase/src/plugin/build.xml?rev=965768&r1=965767&r2=965768&view=diff
==============================================================================
--- nutch/branches/nutchbase/src/plugin/build.xml (original)
+++ nutch/branches/nutchbase/src/plugin/build.xml Tue Jul 20 09:22:55 2010
@@ -43,6 +43,7 @@
<ant dir="parse-tika" target="deploy"/>
<ant dir="scoring-opic" target="deploy"/>
<ant dir="subcollection" target="deploy"/>
+ <ant dir="tld" target="deploy"/>
<ant dir="urlfilter-automaton" target="deploy"/>
<ant dir="urlfilter-domain" target="deploy" />
<ant dir="urlfilter-prefix" target="deploy"/>
@@ -61,7 +62,6 @@
<ant dir="parse-zip" target="deploy"/>
<ant dir="protocol-httpclient" target="deploy"/>
<ant dir="scoring-link" target="deploy"/>
- <ant dir="tld" target="deploy"/>
-->
</target>
Modified:
nutch/branches/nutchbase/src/plugin/tld/src/java/org/apache/nutch/indexer/tld/TLDIndexingFilter.java
URL:
http://svn.apache.org/viewvc/nutch/branches/nutchbase/src/plugin/tld/src/java/org/apache/nutch/indexer/tld/TLDIndexingFilter.java?rev=965768&r1=965767&r2=965768&view=diff
==============================================================================
---
nutch/branches/nutchbase/src/plugin/tld/src/java/org/apache/nutch/indexer/tld/TLDIndexingFilter.java
(original)
+++
nutch/branches/nutchbase/src/plugin/tld/src/java/org/apache/nutch/indexer/tld/TLDIndexingFilter.java
Tue Jul 20 09:22:55 2010
@@ -27,7 +27,6 @@ import org.apache.hadoop.conf.Configurat
import org.apache.nutch.indexer.IndexingException;
import org.apache.nutch.indexer.IndexingFilter;
import org.apache.nutch.indexer.NutchDocument;
-import org.apache.nutch.indexer.lucene.LuceneWriter;
import org.apache.nutch.storage.WebPage;
import org.apache.nutch.storage.WebPage.Field;
import org.apache.nutch.util.URLUtil;
Modified:
nutch/branches/nutchbase/src/plugin/tld/src/java/org/apache/nutch/scoring/tld/TLDScoringFilter.java
URL:
http://svn.apache.org/viewvc/nutch/branches/nutchbase/src/plugin/tld/src/java/org/apache/nutch/scoring/tld/TLDScoringFilter.java?rev=965768&r1=965767&r2=965768&view=diff
==============================================================================
---
nutch/branches/nutchbase/src/plugin/tld/src/java/org/apache/nutch/scoring/tld/TLDScoringFilter.java
(original)
+++
nutch/branches/nutchbase/src/plugin/tld/src/java/org/apache/nutch/scoring/tld/TLDScoringFilter.java
Tue Jul 20 09:22:55 2010
@@ -17,97 +17,91 @@
package org.apache.nutch.scoring.tld;
-import java.util.List;
import java.util.Collection;
-import java.util.Map.Entry;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.io.Text;
-import org.apache.nutch.crawl.CrawlDatum;
-import org.apache.nutch.crawl.Inlinks;
import org.apache.nutch.indexer.NutchDocument;
-import org.apache.nutch.parse.Parse;
-import org.apache.nutch.parse.ParseData;
-import org.apache.nutch.protocol.Content;
+import org.apache.nutch.scoring.ScoreDatum;
import org.apache.nutch.scoring.ScoringFilter;
import org.apache.nutch.scoring.ScoringFilterException;
+import org.apache.nutch.storage.WebPage;
import org.apache.nutch.util.domain.DomainSuffix;
import org.apache.nutch.util.domain.DomainSuffixes;
-
/**
* Scoring filter to boost tlds.
+ *
* @author Enis Soztutar <[email protected]>
*/
public class TLDScoringFilter implements ScoringFilter {
- private Configuration conf;
- private DomainSuffixes tldEntries;
+ private Configuration conf;
+ private DomainSuffixes tldEntries;
+
+ private final static Set<WebPage.Field> FIELDS = new
HashSet<WebPage.Field>();
- public TLDScoringFilter() {
- tldEntries = DomainSuffixes.getInstance();
- }
-
- public float indexerScore(Text url, NutchDocument doc, CrawlDatum dbDatum,
- CrawlDatum fetchDatum, Parse parse, Inlinks inlinks, float initScore)
- throws ScoringFilterException {
-
- List<String> tlds = doc.getFieldValues("tld");
- float boost = 1.0f;
-
- if(tlds != null) {
- for(String tld : tlds) {
- DomainSuffix entry = tldEntries.get(tld);
- if(entry != null)
- boost *= entry.getBoost();
- }
- }
- return initScore * boost;
- }
-
- public CrawlDatum distributeScoreToOutlink(Text fromUrl, Text toUrl,
- ParseData parseData, CrawlDatum target, CrawlDatum adjust, int allCount,
- int validCount) throws ScoringFilterException {
- return adjust;
- }
-
- public float generatorSortValue(Text url, CrawlDatum datum, float initSort)
- throws ScoringFilterException {
- return initSort;
- }
-
- public void initialScore(Text url, CrawlDatum datum)
- throws ScoringFilterException {
- }
-
- public void injectedScore(Text url, CrawlDatum datum)
- throws ScoringFilterException {
- }
-
- public void passScoreAfterParsing(Text url, Content content, Parse parse)
- throws ScoringFilterException {
- }
-
- public void passScoreBeforeParsing(Text url, CrawlDatum datum, Content
content)
- throws ScoringFilterException {
- }
-
- public void updateDbScore(Text url, CrawlDatum old,
- CrawlDatum datum, List<CrawlDatum> inlinked)
- throws ScoringFilterException {
- }
-
- public Configuration getConf() {
- return conf;
- }
-
- public void setConf(Configuration conf) {
- this.conf = conf;
- }
- public CrawlDatum distributeScoreToOutlinks(Text fromUrl, ParseData
parseData,
- Collection<Entry<Text, CrawlDatum>> targets, CrawlDatum adjust,
- int allCount) throws ScoringFilterException {
- return adjust;
- }
+ public TLDScoringFilter() {
+ tldEntries = DomainSuffixes.getInstance();
+ }
+
+ public Configuration getConf() {
+ return conf;
+ }
+
+ public void setConf(Configuration conf) {
+ this.conf = conf;
+ }
+
+ @Override
+ public Collection<WebPage.Field> getFields() {
+ return FIELDS;
+ }
+
+ @Override
+ public void injectedScore(String url, WebPage page)
+ throws ScoringFilterException {
+ }
+
+ @Override
+ public void initialScore(String url, WebPage page)
+ throws ScoringFilterException {
+
+ }
+
+ @Override
+ public float generatorSortValue(String url, WebPage page, float
initSort)
+ throws ScoringFilterException {
+ return initSort;
+ }
+
+ @Override
+ public void distributeScoreToOutlinks(String fromUrl, WebPage page,
+ Collection<ScoreDatum> scoreData, int allCount)
+ throws ScoringFilterException {
+ }
+
+ @Override
+ public void updateScore(String url, WebPage page,
+ List<ScoreDatum> inlinkedScoreData) throws
ScoringFilterException {
+ }
+
+ @Override
+ public float indexerScore(String url, NutchDocument doc, WebPage page,
+ float initScore) throws ScoringFilterException {
+ List<String> tlds = doc.getFieldValues("tld");
+ float boost = 1.0f;
+
+ if (tlds != null) {
+ for (String tld : tlds) {
+ DomainSuffix entry = tldEntries.get(tld);
+ if (entry != null)
+ boost *= entry.getBoost();
+ }
+ }
+ return initScore * boost;
+ }
}
