svn commit: r983472 - in /nutch/trunk: CHANGES.txt src/plugin/parse-ext/plugin.xml src/plugin/parse-ext/src/java/org/apache/nutch/parse/ext/ExtParser.java

[mattmann]

Author: mattmann
Date: Sun Aug  8 19:02:47 2010
New Revision: 983472

URL: http://svn.apache.org/viewvc?rev=983472&view=rev
Log:
- fix for NUTCH-564 External parser supports encoding attribute

Modified:
    nutch/trunk/CHANGES.txt
    nutch/trunk/src/plugin/parse-ext/plugin.xml
    
nutch/trunk/src/plugin/parse-ext/src/java/org/apache/nutch/parse/ext/ExtParser.java

Modified: nutch/trunk/CHANGES.txt
URL: 
http://svn.apache.org/viewvc/nutch/trunk/CHANGES.txt?rev=983472&r1=983471&r2=983472&view=diff
==============================================================================
--- nutch/trunk/CHANGES.txt (original)
+++ nutch/trunk/CHANGES.txt Sun Aug  8 19:02:47 2010
@@ -2,6 +2,8 @@ Nutch Change Log
 
 Release 2.0 - Current Development
 
+* NUTCH-564 External parser supports encoding attribute (Antony Bowesman, 
mattmann)
+
 * NUTCH-873 Ivy configuration settings don't include Gora (mattmann)
 
 * NUTCH-870 Injector should add the metadata before calling injectedScore 
(jnioche via mattmann)

Modified: nutch/trunk/src/plugin/parse-ext/plugin.xml
URL: 
http://svn.apache.org/viewvc/nutch/trunk/src/plugin/parse-ext/plugin.xml?rev=983472&r1=983471&r2=983472&view=diff
==============================================================================
--- nutch/trunk/src/plugin/parse-ext/plugin.xml (original)
+++ nutch/trunk/src/plugin/parse-ext/plugin.xml Sun Aug  8 19:02:47 2010
@@ -41,6 +41,8 @@
         <parameter name="pathSuffix"  value=""/>
         <parameter name="command"     
value="./build/plugins/parse-ext/command"/>
         <parameter name="timeout"     value="10"/>
+        <!-- can optionally specify an encoding parameter now, see NUTCH-564-->
+        <!-- <parameter name="encoding" value="UTF-8"/> -->
       </implementation>
 
       <implementation id="ExtParser"
@@ -49,6 +51,8 @@
         <parameter name="pathSuffix"  value=""/>
         <parameter name="command"     
value="./build/plugins/parse-ext/command"/>
         <parameter name="timeout"     value="20"/>
+        <!-- can optionally specify an encoding parameter now, see NUTCH-564-->
+        <!-- <parameter name="encoding" value="UTF-8"/> -->
       </implementation>
 
    </extension>

Modified: 
nutch/trunk/src/plugin/parse-ext/src/java/org/apache/nutch/parse/ext/ExtParser.java
URL: 
http://svn.apache.org/viewvc/nutch/trunk/src/plugin/parse-ext/src/java/org/apache/nutch/parse/ext/ExtParser.java?rev=983472&r1=983471&r2=983472&view=diff
==============================================================================
--- 
nutch/trunk/src/plugin/parse-ext/src/java/org/apache/nutch/parse/ext/ExtParser.java
 (original)
+++ 
nutch/trunk/src/plugin/parse-ext/src/java/org/apache/nutch/parse/ext/ExtParser.java
 Sun Aug  8 19:02:47 2010
@@ -42,6 +42,7 @@ import java.util.Hashtable;
 
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
+import java.nio.charset.Charset;
 
 /**
  * A wrapper that invokes external command to do real parsing job.
@@ -57,7 +58,7 @@ public class ExtParser implements Parser
 
   static final int TIMEOUT_DEFAULT = 30; // in seconds
 
-  // handy map from String contentType to String[] {command, timeoutString}
+  // handy map from String contentType to String[] {command, timeoutString, 
encoding}
   Hashtable TYPE_PARAMS_MAP = new Hashtable();
 
   private Configuration conf;  
@@ -77,6 +78,7 @@ public class ExtParser implements Parser
 
     String command = params[0];
     int timeout = Integer.parseInt(params[1]);
+    String encoding = params[2];
 
     if (LOG.isTraceEnabled()) {
       LOG.trace("Use "+command+ " with timeout="+timeout+"secs");
@@ -117,7 +119,7 @@ public class ExtParser implements Parser
                         "External command " + command
                         + " failed with error: " + 
es.toString()).getEmptyParseResult(content.getUrl(), getConf());
 
-      text = os.toString();
+      text = os.toString(encoding);
 
     } catch (Exception e) { // run time exception
       return new ParseStatus(e).getEmptyParseResult(content.getUrl(), 
getConf());
@@ -143,7 +145,7 @@ public class ExtParser implements Parser
     Extension[] extensions = PluginRepository.get(conf).getExtensionPoint(
         "org.apache.nutch.parse.Parser").getExtensions();
 
-    String contentType, command, timeoutString;
+    String contentType, command, timeoutString, encoding;
 
     for (int i = 0; i < extensions.length; i++) {
       Extension extension = extensions[i];
@@ -160,11 +162,16 @@ public class ExtParser implements Parser
       if (command == null || command.equals(""))
         continue;
 
+      // null encoding means default
+      encoding = extension.getAttribute("encoding");
+      if (encoding == null)
+          encoding = Charset.defaultCharset().name();
+
       timeoutString = extension.getAttribute("timeout");
       if (timeoutString == null || timeoutString.equals(""))
         timeoutString = "" + TIMEOUT_DEFAULT;
 
-      TYPE_PARAMS_MAP.put(contentType, new String[] { command, timeoutString 
});
+      TYPE_PARAMS_MAP.put(contentType, new String[] { command, timeoutString, 
encoding });
     }
   }