Author: jnioche
Date: Tue Aug 3 12:29:09 2010
New Revision: 981844
URL: http://svn.apache.org/viewvc?rev=981844&view=rev
Log:
NUTCH-696 Timeout for Parser
Added:
nutch/branches/branch-1.2/src/java/org/apache/nutch/parse/ParseCallable.java
Modified:
nutch/branches/branch-1.2/CHANGES.txt
nutch/branches/branch-1.2/conf/nutch-default.xml
nutch/branches/branch-1.2/src/java/org/apache/nutch/parse/ParseUtil.java
Modified: nutch/branches/branch-1.2/CHANGES.txt
URL:
http://svn.apache.org/viewvc/nutch/branches/branch-1.2/CHANGES.txt?rev=981844&r1=981843&r2=981844&view=diff
==============================================================================
--- nutch/branches/branch-1.2/CHANGES.txt (original)
+++ nutch/branches/branch-1.2/CHANGES.txt Tue Aug 3 12:29:09 2010
@@ -2,6 +2,8 @@ Nutch Change Log
Release 1.2 - Current Development
+* NUTCH-696 Timeout for Parser (ab, jnioche)
+
* NUTCH-857 DistributedBeans should not close their RPC counterparts (kubes)
* NUTCH-855 ScoringFilter and IndexingFilter: To allow for the propagation of
URL Metatags
Modified: nutch/branches/branch-1.2/conf/nutch-default.xml
URL:
http://svn.apache.org/viewvc/nutch/branches/branch-1.2/conf/nutch-default.xml?rev=981844&r1=981843&r2=981844&view=diff
==============================================================================
--- nutch/branches/branch-1.2/conf/nutch-default.xml (original)
+++ nutch/branches/branch-1.2/conf/nutch-default.xml Tue Aug 3 12:29:09 2010
@@ -1064,6 +1064,16 @@
</description>
</property>
+<property>
+ <name>parser.timeout</name>
+ <value>30</value>
+ <description>Timeout in seconds for the parsing of a document, otherwise
treats it as an exception and
+ moves on the the following documents. This parameter is applied to any
Parser implementation.
+ Set to -1 to deactivate, bearing in mind that this could cause
+ the parsing to crash because of a very long or corrupted document.
+ </description>
+</property>
+
<!-- urlfilter plugin properties -->
<property>
Added:
nutch/branches/branch-1.2/src/java/org/apache/nutch/parse/ParseCallable.java
URL:
http://svn.apache.org/viewvc/nutch/branches/branch-1.2/src/java/org/apache/nutch/parse/ParseCallable.java?rev=981844&view=auto
==============================================================================
---
nutch/branches/branch-1.2/src/java/org/apache/nutch/parse/ParseCallable.java
(added)
+++
nutch/branches/branch-1.2/src/java/org/apache/nutch/parse/ParseCallable.java
Tue Aug 3 12:29:09 2010
@@ -0,0 +1,20 @@
+package org.apache.nutch.parse;
+
+import java.util.concurrent.Callable;
+
+import org.apache.nutch.protocol.Content;
+
+class ParseCallable implements Callable<ParseResult> {
+ private Parser p;
+ private Content content;
+
+ public ParseCallable(Parser p, Content content) {
+ this.p = p;
+ this.content = content;
+ }
+
+ @Override
+ public ParseResult call() throws Exception {
+ return p.getParse(content);
+ }
+}
\ No newline at end of file
Modified:
nutch/branches/branch-1.2/src/java/org/apache/nutch/parse/ParseUtil.java
URL:
http://svn.apache.org/viewvc/nutch/branches/branch-1.2/src/java/org/apache/nutch/parse/ParseUtil.java?rev=981844&r1=981843&r2=981844&view=diff
==============================================================================
--- nutch/branches/branch-1.2/src/java/org/apache/nutch/parse/ParseUtil.java
(original)
+++ nutch/branches/branch-1.2/src/java/org/apache/nutch/parse/ParseUtil.java
Tue Aug 3 12:29:09 2010
@@ -17,14 +17,15 @@
package org.apache.nutch.parse;
// Commons Logging imports
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-// Nutch Imports
-import org.apache.nutch.protocol.Content;
+import java.util.concurrent.FutureTask;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
-// Hadoop imports
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
+import org.apache.nutch.protocol.Content;
/**
@@ -41,6 +42,8 @@ public class ParseUtil {
/* our log stream */
public static final Log LOG = LogFactory.getLog(ParseUtil.class);
private ParserFactory parserFactory;
+ /** Parser timeout set to 30 sec by default. Set -1 to deactivate **/
+ private int MAX_PARSE_TIME = 30;
/**
*
@@ -48,6 +51,7 @@ public class ParseUtil {
*/
public ParseUtil(Configuration conf) {
this.parserFactory = new ParserFactory(conf);
+ MAX_PARSE_TIME=conf.getInt("parser.timeout", 30);
}
/**
@@ -79,7 +83,11 @@ public class ParseUtil {
if (LOG.isDebugEnabled()) {
LOG.debug("Parsing [" + content.getUrl() + "] with [" + parsers[i] +
"]");
}
- parseResult = parsers[i].getParse(content);
+ if (MAX_PARSE_TIME!=-1)
+ parseResult = runParser(parsers[i], content);
+ else
+ parseResult = parsers[i].getParse(content);
+
if (parseResult != null && !parseResult.isEmpty())
return parseResult;
}
@@ -125,7 +133,11 @@ public class ParseUtil {
throw new ParseException(e.getMessage());
}
- ParseResult parseResult = p.getParse(content);
+ ParseResult parseResult = null;
+ if (MAX_PARSE_TIME!=-1)
+ parseResult = runParser(p, content);
+ else
+ parseResult = p.getParse(content);
if (parseResult != null && !parseResult.isEmpty()) {
return parseResult;
} else {
@@ -135,6 +147,27 @@ public class ParseUtil {
}
return new ParseStatus(new ParseException("Unable to successfully parse
content")).getEmptyParseResult(content.getUrl(), null);
}
- }
+ }
+
+ private ParseResult runParser(Parser p, Content content) {
+ ParseCallable pc = new ParseCallable(p, content);
+ FutureTask<ParseResult> task = new FutureTask<ParseResult>(pc);
+ ParseResult res = null;
+ Thread t = new Thread(task);
+ t.start();
+ try {
+ res = task.get(MAX_PARSE_TIME, TimeUnit.SECONDS);
+ } catch (TimeoutException e) {
+ LOG.warn("TIMEOUT parsing " + content.getUrl() + " with " + p);
+ } catch (Exception e) {
+ task.cancel(true);
+ res = null;
+ t.interrupt();
+ } finally {
+ t = null;
+ pc = null;
+ }
+ return res;
+ }
}
