Author: ab
Date: Fri Aug 6 14:04:08 2010
New Revision: 982970
URL: http://svn.apache.org/viewvc?rev=982970&view=rev
Log:
NUTCH-858 No longer able to set per-field boosts on lucene documents
Added:
nutch/branches/branch-1.2/src/java/org/apache/nutch/indexer/NutchField.java
Modified:
nutch/branches/branch-1.2/CHANGES.txt
nutch/branches/branch-1.2/src/java/org/apache/nutch/indexer/IndexerMapReduce.java
nutch/branches/branch-1.2/src/java/org/apache/nutch/indexer/NutchDocument.java
nutch/branches/branch-1.2/src/java/org/apache/nutch/indexer/lucene/LuceneWriter.java
nutch/branches/branch-1.2/src/java/org/apache/nutch/indexer/solr/SolrWriter.java
nutch/branches/branch-1.2/src/plugin/tld/src/java/org/apache/nutch/scoring/tld/TLDScoringFilter.java
Modified: nutch/branches/branch-1.2/CHANGES.txt
URL:
http://svn.apache.org/viewvc/nutch/branches/branch-1.2/CHANGES.txt?rev=982970&r1=982969&r2=982970&view=diff
==============================================================================
--- nutch/branches/branch-1.2/CHANGES.txt (original)
+++ nutch/branches/branch-1.2/CHANGES.txt Fri Aug 6 14:04:08 2010
@@ -2,6 +2,8 @@ Nutch Change Log
Release 1.2 - Current Development
+* NUTCH-858 No longer able to set per-field boosts on lucene documents (ab)
+
* NUTCH-869 Add parse-html back (jnioche)
* NUTCH-871 MoreIndexingFilter missing date format (Max Lynch via mattmann)
Modified:
nutch/branches/branch-1.2/src/java/org/apache/nutch/indexer/IndexerMapReduce.java
URL:
http://svn.apache.org/viewvc/nutch/branches/branch-1.2/src/java/org/apache/nutch/indexer/IndexerMapReduce.java?rev=982970&r1=982969&r2=982970&view=diff
==============================================================================
---
nutch/branches/branch-1.2/src/java/org/apache/nutch/indexer/IndexerMapReduce.java
(original)
+++
nutch/branches/branch-1.2/src/java/org/apache/nutch/indexer/IndexerMapReduce.java
Fri Aug 6 14:04:08 2010
@@ -152,7 +152,7 @@ implements Mapper<Text, Writable, Text,
return;
}
// apply boost to all indexed fields.
- doc.setScore(boost);
+ doc.setWeight(boost);
// store boost for use by explain and dedup
doc.add("boost", Float.toString(boost));
Modified:
nutch/branches/branch-1.2/src/java/org/apache/nutch/indexer/NutchDocument.java
URL:
http://svn.apache.org/viewvc/nutch/branches/branch-1.2/src/java/org/apache/nutch/indexer/NutchDocument.java?rev=982970&r1=982969&r2=982970&view=diff
==============================================================================
---
nutch/branches/branch-1.2/src/java/org/apache/nutch/indexer/NutchDocument.java
(original)
+++
nutch/branches/branch-1.2/src/java/org/apache/nutch/indexer/NutchDocument.java
Fri Aug 6 14:04:08 2010
@@ -19,11 +19,9 @@ package org.apache.nutch.indexer;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
-import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.Iterator;
-import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
@@ -35,51 +33,48 @@ import org.apache.nutch.metadata.Metadat
/** A {...@link NutchDocument} is the unit of indexing.*/
public class NutchDocument
-implements Writable, Iterable<Entry<String, List<String>>> {
+implements Writable, Iterable<Entry<String, NutchField>> {
- public static final byte VERSION = 1;
-
- private Map<String, List<String>> fields;
+ public static final byte VERSION = 2;
+
+ private Map<String, NutchField> fields;
private Metadata documentMeta;
- private float score;
+ private float weight;
public NutchDocument() {
- fields = new HashMap<String, List<String>>();
+ fields = new HashMap<String, NutchField>();
documentMeta = new Metadata();
- score = 0.0f;
+ weight = 1.0f;
}
- public void add(String name, String value) {
- List<String> fieldValues = fields.get(name);
- if (fieldValues == null) {
- fieldValues = new ArrayList<String>();
+ public void add(String name, Object value) {
+ NutchField field = fields.get(name);
+ if (field == null) {
+ field = new NutchField(value);
+ fields.put(name, field);
+ } else {
+ field.add(value);
}
- fieldValues.add(value);
- fields.put(name, fieldValues);
- }
-
- private void addFieldUnprotected(String name, String value) {
- fields.get(name).add(value);
}
- public String getFieldValue(String name) {
- List<String> fieldValues = fields.get(name);
- if (fieldValues == null) {
+ public Object getFieldValue(String name) {
+ NutchField field = fields.get(name);
+ if (field == null) {
return null;
}
- if (fieldValues.size() == 0) {
+ if (field.getValues().size() == 0) {
return null;
}
- return fieldValues.get(0);
+ return field.getValues().get(0);
}
- public List<String> getFieldValues(String name) {
+ public NutchField getField(String name) {
return fields.get(name);
}
- public List<String> removeField(String name) {
+ public NutchField removeField(String name) {
return fields.remove(name);
}
@@ -88,16 +83,16 @@ implements Writable, Iterable<Entry<Stri
}
/** Iterate over all fields. */
- public Iterator<Entry<String, List<String>>> iterator() {
+ public Iterator<Entry<String, NutchField>> iterator() {
return fields.entrySet().iterator();
}
- public float getScore() {
- return score;
+ public float getWeight() {
+ return weight;
}
- public void setScore(float score) {
- this.score = score;
+ public void setWeight(float weight) {
+ this.weight = weight;
}
public Metadata getDocumentMeta() {
@@ -105,6 +100,7 @@ implements Writable, Iterable<Entry<Stri
}
public void readFields(DataInput in) throws IOException {
+ fields.clear();
byte version = in.readByte();
if (version != VERSION) {
throw new VersionMismatchException(VERSION, version);
@@ -112,30 +108,23 @@ implements Writable, Iterable<Entry<Stri
int size = WritableUtils.readVInt(in);
for (int i = 0; i < size; i++) {
String name = Text.readString(in);
- int numValues = WritableUtils.readVInt(in);
- fields.put(name, new ArrayList<String>());
- for (int j = 0; j < numValues; j++) {
- String value = Text.readString(in);
- addFieldUnprotected(name, value);
- }
+ NutchField field = new NutchField();
+ field.readFields(in);
+ fields.put(name, field);
}
- score = in.readFloat();
+ weight = in.readFloat();
documentMeta.readFields(in);
}
public void write(DataOutput out) throws IOException {
out.writeByte(VERSION);
WritableUtils.writeVInt(out, fields.size());
- for (Map.Entry<String, List<String>> entry : fields.entrySet()) {
+ for (Map.Entry<String, NutchField> entry : fields.entrySet()) {
Text.writeString(out, entry.getKey());
- List<String> values = entry.getValue();
- WritableUtils.writeVInt(out, values.size());
- for (String value : values) {
- Text.writeString(out, value);
- }
+ NutchField field = entry.getValue();
+ field.write(out);
}
- out.writeFloat(score);
+ out.writeFloat(weight);
documentMeta.write(out);
}
-
}
Added:
nutch/branches/branch-1.2/src/java/org/apache/nutch/indexer/NutchField.java
URL:
http://svn.apache.org/viewvc/nutch/branches/branch-1.2/src/java/org/apache/nutch/indexer/NutchField.java?rev=982970&view=auto
==============================================================================
--- nutch/branches/branch-1.2/src/java/org/apache/nutch/indexer/NutchField.java
(added)
+++ nutch/branches/branch-1.2/src/java/org/apache/nutch/indexer/NutchField.java
Fri Aug 6 14:04:08 2010
@@ -0,0 +1,64 @@
+package org.apache.nutch.indexer;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
+import org.apache.hadoop.io.Writable;
+
+/**
+ * This class represents a multi-valued field with a weight. Values are
arbitrary
+ * objects.
+ */
+public class NutchField implements Writable {
+ private float weight;
+ private List<Object> values = new ArrayList<Object>();
+
+ public NutchField() {
+
+ }
+
+ public NutchField(Object value) {
+ this(value, 1.0f);
+ }
+
+ public NutchField(Object value, float weight) {
+ this.weight = weight;
+ if (value instanceof Collection) {
+ values.addAll((Collection<Object>)value);
+ } else {
+ values.add(value);
+ }
+ }
+
+ public void add(Object value) {
+ values.add(value);
+ }
+
+ public float getWeight() {
+ return weight;
+ }
+
+ public void setWeight(float weight) {
+ this.weight = weight;
+ }
+
+ public List<Object> getValues() {
+ return values;
+ }
+
+ public void reset() {
+ weight = 1.0f;
+ values.clear();
+ }
+
+ public void readFields(DataInput in) throws IOException {
+ }
+
+ public void write(DataOutput out) throws IOException {
+ }
+
+}
Modified:
nutch/branches/branch-1.2/src/java/org/apache/nutch/indexer/lucene/LuceneWriter.java
URL:
http://svn.apache.org/viewvc/nutch/branches/branch-1.2/src/java/org/apache/nutch/indexer/lucene/LuceneWriter.java?rev=982970&r1=982969&r2=982970&view=diff
==============================================================================
---
nutch/branches/branch-1.2/src/java/org/apache/nutch/indexer/lucene/LuceneWriter.java
(original)
+++
nutch/branches/branch-1.2/src/java/org/apache/nutch/indexer/lucene/LuceneWriter.java
Fri Aug 6 14:04:08 2010
@@ -40,6 +40,7 @@ import org.apache.nutch.analysis.NutchAn
import org.apache.nutch.analysis.NutchDocumentAnalyzer;
import org.apache.nutch.indexer.Indexer;
import org.apache.nutch.indexer.NutchDocument;
+import org.apache.nutch.indexer.NutchField;
import org.apache.nutch.indexer.NutchIndexWriter;
import org.apache.nutch.indexer.NutchSimilarity;
import org.apache.nutch.metadata.Metadata;
@@ -78,10 +79,10 @@ public class LuceneWriter implements Nut
private Document createLuceneDoc(NutchDocument doc) {
final Document out = new Document();
- out.setBoost(doc.getScore());
+ out.setBoost(doc.getWeight());
final Metadata documentMeta = doc.getDocumentMeta();
- for (final Entry<String, List<String>> entry : doc) {
+ for (final Entry<String, NutchField> entry : doc) {
final String fieldName = entry.getKey();
Field.Store store = fieldStore.get(fieldName);
@@ -132,8 +133,10 @@ public class LuceneWriter implements Nut
}
}
- for (final String fieldValue : entry.getValue()) {
- out.add(new Field(fieldName, fieldValue, store, index, vector));
+ for (final Object fieldValue : entry.getValue().getValues()) {
+ Field f = new Field(fieldName, fieldValue.toString(), store, index,
vector);
+ f.setBoost(entry.getValue().getWeight());
+ out.add(f);
}
}
Modified:
nutch/branches/branch-1.2/src/java/org/apache/nutch/indexer/solr/SolrWriter.java
URL:
http://svn.apache.org/viewvc/nutch/branches/branch-1.2/src/java/org/apache/nutch/indexer/solr/SolrWriter.java?rev=982970&r1=982969&r2=982970&view=diff
==============================================================================
---
nutch/branches/branch-1.2/src/java/org/apache/nutch/indexer/solr/SolrWriter.java
(original)
+++
nutch/branches/branch-1.2/src/java/org/apache/nutch/indexer/solr/SolrWriter.java
Fri Aug 6 14:04:08 2010
@@ -23,6 +23,7 @@ import java.util.Map.Entry;
import org.apache.hadoop.mapred.JobConf;
import org.apache.nutch.indexer.NutchDocument;
+import org.apache.nutch.indexer.NutchField;
import org.apache.nutch.indexer.NutchIndexWriter;
import org.apache.solr.client.solrj.SolrServer;
import org.apache.solr.client.solrj.SolrServerException;
@@ -47,16 +48,16 @@ public class SolrWriter implements Nutch
public void write(NutchDocument doc) throws IOException {
final SolrInputDocument inputDoc = new SolrInputDocument();
- for(final Entry<String, List<String>> e : doc) {
- for (final String val : e.getValue()) {
- inputDoc.addField(solrMapping.mapKey(e.getKey()), val);
+ for(final Entry<String, NutchField> e : doc) {
+ for (final Object val : e.getValue().getValues()) {
+ inputDoc.addField(solrMapping.mapKey(e.getKey()), val,
e.getValue().getWeight());
String sCopy = solrMapping.mapCopyKey(e.getKey());
if (sCopy != e.getKey()) {
- inputDoc.addField(sCopy, val);
+ inputDoc.addField(sCopy, val, e.getValue().getWeight());
}
}
}
- inputDoc.setDocumentBoost(doc.getScore());
+ inputDoc.setDocumentBoost(doc.getWeight());
inputDocs.add(inputDoc);
if (inputDocs.size() > commitSize) {
try {
Modified:
nutch/branches/branch-1.2/src/plugin/tld/src/java/org/apache/nutch/scoring/tld/TLDScoringFilter.java
URL:
http://svn.apache.org/viewvc/nutch/branches/branch-1.2/src/plugin/tld/src/java/org/apache/nutch/scoring/tld/TLDScoringFilter.java?rev=982970&r1=982969&r2=982970&view=diff
==============================================================================
---
nutch/branches/branch-1.2/src/plugin/tld/src/java/org/apache/nutch/scoring/tld/TLDScoringFilter.java
(original)
+++
nutch/branches/branch-1.2/src/plugin/tld/src/java/org/apache/nutch/scoring/tld/TLDScoringFilter.java
Fri Aug 6 14:04:08 2010
@@ -26,6 +26,7 @@ import org.apache.hadoop.io.Text;
import org.apache.nutch.crawl.CrawlDatum;
import org.apache.nutch.crawl.Inlinks;
import org.apache.nutch.indexer.NutchDocument;
+import org.apache.nutch.indexer.NutchField;
import org.apache.nutch.parse.Parse;
import org.apache.nutch.parse.ParseData;
import org.apache.nutch.protocol.Content;
@@ -52,12 +53,12 @@ public class TLDScoringFilter implements
CrawlDatum fetchDatum, Parse parse, Inlinks inlinks, float initScore)
throws ScoringFilterException {
- List<String> tlds = doc.getFieldValues("tld");
+ NutchField tlds = doc.getField("tld");
float boost = 1.0f;
if(tlds != null) {
- for(String tld : tlds) {
- DomainSuffix entry = tldEntries.get(tld);
+ for(Object tld : tlds.getValues()) {
+ DomainSuffix entry = tldEntries.get(tld.toString());
if(entry != null)
boost *= entry.getBoost();
}
