This is an automated email from the ASF dual-hosted git repository. markus pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/nutch.git
The following commit(s) were added to refs/heads/master by this push: new 9a77f43 NUTCH-2411 Index-metadata to support indexing multiple values for a field 9a77f43 is described below commit 9a77f43774b2c3cd70785895afb989e9ee2d8d5f Author: Markus Jelsma <mar...@apache.org> AuthorDate: Thu Mar 8 14:03:12 2018 +0100 NUTCH-2411 Index-metadata to support indexing multiple values for a field --- conf/nutch-default.xml | 9 ++++++ .../nutch/indexer/metadata/MetadataIndexer.java | 35 ++++++++++++++++++---- 2 files changed, 38 insertions(+), 6 deletions(-) diff --git a/conf/nutch-default.xml b/conf/nutch-default.xml index 87c4058..71ef51b 100644 --- a/conf/nutch-default.xml +++ b/conf/nutch-default.xml @@ -1738,6 +1738,15 @@ visit https://wiki.apache.org/nutch/SimilarityScoringFilter--> </description> </property> +<property> + <name>index.metadata.separator</name> + <value></value> + <description> + Separator to use if you want to index multiple values for a given field. Leave empty to + treat each value as a single value. + </description> +</property> + <!-- index-geoip plugin properties --> <property> <name>index.geoip.usage</name> diff --git a/src/plugin/index-metadata/src/java/org/apache/nutch/indexer/metadata/MetadataIndexer.java b/src/plugin/index-metadata/src/java/org/apache/nutch/indexer/metadata/MetadataIndexer.java index edb8b15..74d9eb1 100644 --- a/src/plugin/index-metadata/src/java/org/apache/nutch/indexer/metadata/MetadataIndexer.java +++ b/src/plugin/index-metadata/src/java/org/apache/nutch/indexer/metadata/MetadataIndexer.java @@ -17,9 +17,12 @@ package org.apache.nutch.indexer.metadata; +import java.util.Arrays; import java.util.HashMap; +import java.util.HashSet; import java.util.Locale; import java.util.Map; +import java.util.Set; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.Text; @@ -35,17 +38,21 @@ import org.apache.nutch.parse.Parse; * Indexer which can be configured to extract metadata from the crawldb, parse * metadata or content metadata. You can specify the properties "index.db.md", * "index.parse.md" or "index.content.md" who's values are comma-delimited - * Example : key1,key2,key3. + * <value>key1,key2,key3</value>. */ public class MetadataIndexer implements IndexingFilter { private Configuration conf; private String[] dbFieldnames; private Map<String, String> parseFieldnames; private String[] contentFieldnames; + private String separator; + private Set<String> mvFields; private static final String db_CONF_PROPERTY = "index.db.md"; private static final String parse_CONF_PROPERTY = "index.parse.md"; private static final String content_CONF_PROPERTY = "index.content.md"; - + private static final String separator_CONF_PROPERTY = "index.metadata.separator"; + private static final String mvfields_CONF_PROPERTY = "index.metadata.multivalued.fields"; + public NutchDocument filter(NutchDocument doc, Parse parse, Text url, CrawlDatum datum, Inlinks inlinks) throws IndexingException { @@ -58,7 +65,7 @@ public class MetadataIndexer implements IndexingFilter { for (String metatag : dbFieldnames) { Writable metadata = datum.getMetaData().get(new Text(metatag)); if (metadata != null) - doc.add(metatag, metadata.toString()); + add(doc, metatag, metadata.toString()); } } @@ -67,7 +74,7 @@ public class MetadataIndexer implements IndexingFilter { for (String metatag : parseFieldnames.keySet()) { for (String value : parse.getData().getParseMeta().getValues(metatag)) { if (value != null) - doc.add(parseFieldnames.get(metatag), value); + add(doc, parseFieldnames.get(metatag), value); } } } @@ -77,13 +84,27 @@ public class MetadataIndexer implements IndexingFilter { for (String metatag : contentFieldnames) { for (String value : parse.getData().getContentMeta().getValues(metatag)) { if (value != null) - doc.add(metatag, value); + add(doc, metatag, value); } } } return doc; } + + protected void add(NutchDocument doc, String key, String value) { + if (separator == null || value.indexOf(separator) == -1 || !mvFields.contains(key)) { + doc.add(key, value); + } else { + String[] parts = value.split(separator); + for (String part : parts) { + part = part.trim(); + if (part.length() != 0) { + doc.add(key, part); + } + } + } + } public void setConf(Configuration conf) { this.conf = conf; @@ -93,7 +114,9 @@ public class MetadataIndexer implements IndexingFilter { parseFieldnames.put(metatag.toLowerCase(Locale.ROOT), metatag); } contentFieldnames = conf.getStrings(content_CONF_PROPERTY); - + + separator = conf.get(separator_CONF_PROPERTY, null); + mvFields = new HashSet(Arrays.asList(conf.getStrings(mvfields_CONF_PROPERTY, new String[0]))); // TODO check conflict between field names e.g. could have same label // from different sources -- To stop receiving notification emails like this one, please contact mar...@apache.org.