Author: mattmann
Date: Sun Jul 25 06:30:31 2010
New Revision: 978988
URL: http://svn.apache.org/viewvc?rev=978988&view=rev
Log:
- fix for NUTCH-677 Segment merge filtering based on segment content
Added:
nutch/trunk/src/java/org/apache/nutch/segment/SegmentMergeFilter.java
(with props)
nutch/trunk/src/java/org/apache/nutch/segment/SegmentMergeFilters.java
(with props)
Modified:
nutch/trunk/CHANGES.txt
nutch/trunk/src/java/org/apache/nutch/segment/SegmentMerger.java
nutch/trunk/src/plugin/nutch-extensionpoints/plugin.xml
Modified: nutch/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/nutch/trunk/CHANGES.txt?rev=978988&r1=978987&r2=978988&view=diff
==============================================================================
--- nutch/trunk/CHANGES.txt (original)
+++ nutch/trunk/CHANGES.txt Sun Jul 25 06:30:31 2010
@@ -2,6 +2,8 @@ Nutch Change Log
Release 2.0 - Current Development
+* NUTCH-677 Segment merge filtering based on segment content (Marcin
Okraszewski via mattmann)
+
* NUTCH-774 Retry interval in crawl date is set to 0 (Reinhard Schwab via
mattmann)
* NUTCH-697 Generate log output for solr indexer and dedup (Dmitry Lihachev,
Jeroen van Vianen via mattmann)
Added: nutch/trunk/src/java/org/apache/nutch/segment/SegmentMergeFilter.java
URL:
http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/segment/SegmentMergeFilter.java?rev=978988&view=auto
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/segment/SegmentMergeFilter.java
(added)
+++ nutch/trunk/src/java/org/apache/nutch/segment/SegmentMergeFilter.java Sun
Jul 25 06:30:31 2010
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nutch.segment;
+
+import java.util.Collection;
+
+import org.apache.hadoop.io.WritableComparable;
+import org.apache.nutch.crawl.CrawlDatum;
+import org.apache.nutch.parse.ParseData;
+import org.apache.nutch.parse.ParseText;
+import org.apache.nutch.protocol.Content;
+
+/**
+ * Interface used to filter segments during segment merge. It allows filtering
+ * on more sophisticated criteria than just URLs. In particular it allows
+ * filtering based on metadata collected while parsing page.
+ *
+ */
+public interface SegmentMergeFilter {
+ /** The name of the extension point. */
+ public final static String X_POINT_ID = SegmentMergeFilter.class.getName();
+
+ /**
+ * The filtering method which gets all information being merged for a given
+ * key (URL).
+ *
+ * @return <tt>true</tt> values for this <tt>key</tt> (URL) should be merged
+ * into the new segment.
+ */
+ public boolean filter(WritableComparable key, CrawlDatum generateData,
+ CrawlDatum fetchData, CrawlDatum sigData, Content content,
+ ParseData parseData, ParseText parseText, Collection<CrawlDatum> linked);
+}
Propchange:
nutch/trunk/src/java/org/apache/nutch/segment/SegmentMergeFilter.java
------------------------------------------------------------------------------
svn:eol-style = native
Added: nutch/trunk/src/java/org/apache/nutch/segment/SegmentMergeFilters.java
URL:
http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/segment/SegmentMergeFilters.java?rev=978988&view=auto
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/segment/SegmentMergeFilters.java
(added)
+++ nutch/trunk/src/java/org/apache/nutch/segment/SegmentMergeFilters.java Sun
Jul 25 06:30:31 2010
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nutch.segment;
+
+import java.util.Collection;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.WritableComparable;
+import org.apache.nutch.crawl.CrawlDatum;
+import org.apache.nutch.net.URLFilter;
+import org.apache.nutch.parse.ParseData;
+import org.apache.nutch.parse.ParseText;
+import org.apache.nutch.plugin.Extension;
+import org.apache.nutch.plugin.ExtensionPoint;
+import org.apache.nutch.plugin.PluginRepository;
+import org.apache.nutch.plugin.PluginRuntimeException;
+import org.apache.nutch.protocol.Content;
+
+/**
+ * This class wraps all {...@link SegmentMergeFilter} extensions in a single
object
+ * so it is easier to operate on them. If any of extensions returns
+ * <tt>false</tt> this one will return <tt>false</tt> as well.
+ *
+ */
+public class SegmentMergeFilters {
+ private static final Log LOG = LogFactory.getLog(SegmentMergeFilters.class);
+ private SegmentMergeFilter[] filters;
+
+ public SegmentMergeFilters(Configuration conf) {
+ try {
+ ExtensionPoint point = PluginRepository.get(conf).getExtensionPoint(
+ SegmentMergeFilter.X_POINT_ID);
+ if (point == null)
+ throw new RuntimeException(URLFilter.X_POINT_ID + " not found.");
+ Extension[] extensions = point.getExtensions();
+ filters = new SegmentMergeFilter[extensions.length];
+ for (int i = 0; i < extensions.length; i++) {
+ filters[i] = (SegmentMergeFilter) extensions[i].getExtensionInstance();
+ }
+ } catch (PluginRuntimeException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ /**
+ * Iterates over all {...@link SegmentMergeFilter} extensions and if any of
them
+ * returns false, it will return false as well.
+ *
+ * @return <tt>true</tt> values for this <tt>key</tt> (URL) should be merged
+ * into the new segment.
+ */
+ public boolean filter(WritableComparable key, CrawlDatum generateData,
+ CrawlDatum fetchData, CrawlDatum sigData, Content content,
+ ParseData parseData, ParseText parseText, Collection<CrawlDatum> linked)
{
+ for (SegmentMergeFilter filter : filters) {
+ if (!filter.filter(key, generateData, fetchData, sigData, content,
+ parseData, parseText, linked)) {
+ if (LOG.isTraceEnabled())
+ LOG
+ .trace("Key " + key + " dropped by "
+ + filter.getClass().getName());
+ return false;
+ }
+ }
+ if (LOG.isTraceEnabled())
+ LOG.trace("Key " + key + " accepted for merge.");
+ return true;
+ }
+}
Propchange:
nutch/trunk/src/java/org/apache/nutch/segment/SegmentMergeFilters.java
------------------------------------------------------------------------------
svn:eol-style = native
Modified: nutch/trunk/src/java/org/apache/nutch/segment/SegmentMerger.java
URL:
http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/segment/SegmentMerger.java?rev=978988&r1=978987&r2=978988&view=diff
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/segment/SegmentMerger.java (original)
+++ nutch/trunk/src/java/org/apache/nutch/segment/SegmentMerger.java Sun Jul 25
06:30:31 2010
@@ -120,6 +120,7 @@ public class SegmentMerger extends Confi
private URLFilters filters = null;
private URLNormalizers normalizers = null;
+ private SegmentMergeFilters mergeFilters = null;
private long sliceSize = -1;
private long curCount = 0;
@@ -312,8 +313,10 @@ public class SegmentMerger extends Confi
public void setConf(Configuration conf) {
super.setConf(conf);
if (conf == null) return;
- if (conf.getBoolean("segment.merger.filter", false))
+ if (conf.getBoolean("segment.merger.filter", false)) {
filters = new URLFilters(conf);
+ mergeFilters = new SegmentMergeFilters(conf);
+ }
if (conf.getBoolean("segment.merger.normalizer", false))
normalizers = new URLNormalizers(conf, URLNormalizers.SCOPE_DEFAULT);
sliceSize = conf.getLong("segment.merger.slice", -1);
@@ -470,6 +473,13 @@ public class SegmentMerger extends Confi
}
}
}
+ // perform filtering based on full merge record
+ if (mergeFilters != null &&
+ !mergeFilters.filter(key, lastG, lastF, lastSig, lastC, lastPD,
lastPT,
+ linked.isEmpty() ? null :
linked.lastEntry().getValue())){
+ return;
+ }
+
curCount++;
String sliceName = null;
MetaWrapper wrapper = new MetaWrapper();
Modified: nutch/trunk/src/plugin/nutch-extensionpoints/plugin.xml
URL:
http://svn.apache.org/viewvc/nutch/trunk/src/plugin/nutch-extensionpoints/plugin.xml?rev=978988&r1=978987&r2=978988&view=diff
==============================================================================
--- nutch/trunk/src/plugin/nutch-extensionpoints/plugin.xml (original)
+++ nutch/trunk/src/plugin/nutch-extensionpoints/plugin.xml Sun Jul 25 06:30:31
2010
@@ -52,4 +52,8 @@
id="org.apache.nutch.scoring.ScoringFilter"
name="Nutch Scoring"/>
+<extension-point
+ id="org.apache.nutch.segment.SegmentMergeFilter"
+ name="Nutch Segment Merge Filter"/>
+
</plugin>
