Author: jnioche
Date: Tue Aug 3 12:14:28 2010
New Revision: 981835
URL: http://svn.apache.org/viewvc?rev=981835&view=rev
Log:
NUTCH-696 Timeout for Parser
Added:
nutch/branches/nutchbase/src/java/org/apache/nutch/parse/ParseCallable.java
Modified:
nutch/branches/nutchbase/CHANGES.txt
nutch/branches/nutchbase/conf/nutch-default.xml
nutch/branches/nutchbase/src/java/org/apache/nutch/parse/ParseUtil.java
Modified: nutch/branches/nutchbase/CHANGES.txt
URL:
http://svn.apache.org/viewvc/nutch/branches/nutchbase/CHANGES.txt?rev=981835&r1=981834&r2=981835&view=diff
==============================================================================
--- nutch/branches/nutchbase/CHANGES.txt (original)
+++ nutch/branches/nutchbase/CHANGES.txt Tue Aug 3 12:14:28 2010
@@ -2,6 +2,8 @@ Nutch Change Log
Release 2.0 - Current Development
+* NUTCH-696 Timeout for Parser (ab, jnioche)
+
* NUTCH-774 Retry interval in crawl date is set to 0 (Reinhard Schwab via
mattmann)
* NUTCH-697 Generate log output for solr indexer and dedup (Dmitry Lihachev,
Jeroen van Vianen via mattmann)
Modified: nutch/branches/nutchbase/conf/nutch-default.xml
URL:
http://svn.apache.org/viewvc/nutch/branches/nutchbase/conf/nutch-default.xml?rev=981835&r1=981834&r2=981835&view=diff
==============================================================================
--- nutch/branches/nutchbase/conf/nutch-default.xml (original)
+++ nutch/branches/nutchbase/conf/nutch-default.xml Tue Aug 3 12:14:28 2010
@@ -925,6 +925,16 @@
</description>
</property>
+<property>
+ <name>parser.timeout</name>
+ <value>30</value>
+ <description>Timeout in seconds for the parsing of a document, otherwise
treats it as an exception and
+ moves on the the following documents. This parameter is applied to any
Parser implementation.
+ Set to -1 to deactivate, bearing in mind that this could cause
+ the parsing to crash because of a very long or corrupted document.
+ </description>
+</property>
+
<!-- urlfilter plugin properties -->
<property>
Added:
nutch/branches/nutchbase/src/java/org/apache/nutch/parse/ParseCallable.java
URL:
http://svn.apache.org/viewvc/nutch/branches/nutchbase/src/java/org/apache/nutch/parse/ParseCallable.java?rev=981835&view=auto
==============================================================================
--- nutch/branches/nutchbase/src/java/org/apache/nutch/parse/ParseCallable.java
(added)
+++ nutch/branches/nutchbase/src/java/org/apache/nutch/parse/ParseCallable.java
Tue Aug 3 12:14:28 2010
@@ -0,0 +1,22 @@
+package org.apache.nutch.parse;
+
+import java.util.concurrent.Callable;
+
+import org.apache.nutch.storage.WebPage;
+
+class ParseCallable implements Callable<Parse> {
+ private Parser p;
+ private WebPage content;
+ private String url;
+
+ public ParseCallable(Parser p, WebPage content, String url) {
+ this.p = p;
+ this.content = content;
+ this.url = url;
+ }
+
+ @Override
+ public Parse call() throws Exception {
+ return p.getParse(url, content);
+ }
+}
\ No newline at end of file
Modified:
nutch/branches/nutchbase/src/java/org/apache/nutch/parse/ParseUtil.java
URL:
http://svn.apache.org/viewvc/nutch/branches/nutchbase/src/java/org/apache/nutch/parse/ParseUtil.java?rev=981835&r1=981834&r2=981835&view=diff
==============================================================================
--- nutch/branches/nutchbase/src/java/org/apache/nutch/parse/ParseUtil.java
(original)
+++ nutch/branches/nutchbase/src/java/org/apache/nutch/parse/ParseUtil.java Tue
Aug 3 12:14:28 2010
@@ -20,6 +20,9 @@ package org.apache.nutch.parse;
import java.net.MalformedURLException;
import java.net.URL;
import java.nio.ByteBuffer;
+import java.util.concurrent.FutureTask;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
import org.apache.avro.util.Utf8;
import org.apache.commons.logging.Log;
@@ -63,7 +66,8 @@ public class ParseUtil extends Configure
private int maxOutlinks;
private boolean ignoreExternalLinks;
private ParserFactory parserFactory;
-
+ /** Parser timeout set to 30 sec by default. Set -1 to deactivate **/
+ private int MAX_PARSE_TIME = 30;
/**
*
* @param conf
@@ -82,6 +86,7 @@ public class ParseUtil extends Configure
public void setConf(Configuration conf) {
this.conf = conf;
parserFactory = new ParserFactory(conf);
+ MAX_PARSE_TIME=conf.getInt("parser.timeout", 30);
sig = SignatureFactory.getSignature(conf);
filters = new URLFilters(conf);
normalizers = new URLNormalizers(conf, URLNormalizers.SCOPE_OUTLINK);
@@ -115,7 +120,13 @@ public class ParseUtil extends Configure
if (LOG.isDebugEnabled()) {
LOG.debug("Parsing [" + url + "] with [" + parsers[i] + "]");
}
- Parse parse = parsers[i].getParse(url, page);
+ Parse parse = null;
+
+ if (MAX_PARSE_TIME!=-1)
+ parse = runParser(parsers[i], url, page);
+ else
+ parse = parsers[i].getParse(url, page);
+
if (ParseStatusUtils.isSuccess(parse.getParseStatus())) {
return parse;
}
@@ -125,6 +136,27 @@ public class ParseUtil extends Configure
" of type " + contentType);
return ParseStatusUtils.getEmptyParse(new ParseException("Unable to
successfully parse content"), null);
}
+
+ private Parse runParser(Parser p, String url, WebPage page) {
+ ParseCallable pc = new ParseCallable(p, page, url);
+ FutureTask<Parse> task = new FutureTask<Parse>(pc);
+ Parse res = null;
+ Thread t = new Thread(task);
+ t.start();
+ try {
+ res = task.get(MAX_PARSE_TIME, TimeUnit.SECONDS);
+ } catch (TimeoutException e) {
+ LOG.warn("TIMEOUT parsing " + url + " with " + p);
+ } catch (Exception e) {
+ task.cancel(true);
+ res = null;
+ t.interrupt();
+ } finally {
+ t = null;
+ pc = null;
+ }
+ return res;
+ }
/**
* Parses given web page and stores parsed content within page. Returns
