Author: markus
Date: Thu Nov 10 15:02:04 2011
New Revision: 1200360

URL: http://svn.apache.org/viewvc?rev=1200360&view=rev
Log:
NUTCH-1061 Migrate MoreIndexingFilter from Apache ORO to java.util.regex

Modified:
    nutch/trunk/CHANGES.txt
    nutch/trunk/src/plugin/index-more/plugin.xml
    
nutch/trunk/src/plugin/index-more/src/java/org/apache/nutch/indexer/more/MoreIndexingFilter.java
    
nutch/trunk/src/plugin/index-more/src/test/org/apache/nutch/indexer/more/TestMoreIndexingFilter.java

Modified: nutch/trunk/CHANGES.txt
URL: 
http://svn.apache.org/viewvc/nutch/trunk/CHANGES.txt?rev=1200360&r1=1200359&r2=1200360&view=diff
==============================================================================
--- nutch/trunk/CHANGES.txt (original)
+++ nutch/trunk/CHANGES.txt Thu Nov 10 15:02:04 2011
@@ -1,5 +1,7 @@
 Nutch Change Log
 
+* NUTCH-1061 Migrate MoreIndexingFilter from Apache ORO to java.util.regex 
(markus)
+
 * NUTCH-1178 Incorrect CSV header CrawlDatumCsvOutputFormat (markus)
 
 * NUTCH-1142 Normalization and filtering in WebGraph (markus)

Modified: nutch/trunk/src/plugin/index-more/plugin.xml
URL: 
http://svn.apache.org/viewvc/nutch/trunk/src/plugin/index-more/plugin.xml?rev=1200360&r1=1200359&r2=1200360&view=diff
==============================================================================
--- nutch/trunk/src/plugin/index-more/plugin.xml (original)
+++ nutch/trunk/src/plugin/index-more/plugin.xml Thu Nov 10 15:02:04 2011
@@ -39,14 +39,4 @@
                       
class="org.apache.nutch.indexer.more.MoreIndexingFilter"/>
    </extension>
 
-   <!--
-   <extension id="org.apache.nutch.searcher.more.MoreQueryFilter"
-              name="Nutch More Query Filter"
-              point="org.apache.nutch.searcher.QueryFilter">
-      <implementation id="MoreQueryFilter"
-                      class="org.apache.nutch.searcher.more.MoreQueryFilter"
-                      raw-fields="more"/>
-   </extension>
-   -->
-
 </plugin>

Modified: 
nutch/trunk/src/plugin/index-more/src/java/org/apache/nutch/indexer/more/MoreIndexingFilter.java
URL: 
http://svn.apache.org/viewvc/nutch/trunk/src/plugin/index-more/src/java/org/apache/nutch/indexer/more/MoreIndexingFilter.java?rev=1200360&r1=1200359&r2=1200360&view=diff
==============================================================================
--- 
nutch/trunk/src/plugin/index-more/src/java/org/apache/nutch/indexer/more/MoreIndexingFilter.java
 (original)
+++ 
nutch/trunk/src/plugin/index-more/src/java/org/apache/nutch/indexer/more/MoreIndexingFilter.java
 Thu Nov 10 15:02:04 2011
@@ -16,13 +16,6 @@
  */
 package org.apache.nutch.indexer.more;
 
-
-import org.apache.oro.text.regex.Perl5Compiler;
-import org.apache.oro.text.regex.Perl5Matcher;
-import org.apache.oro.text.regex.Perl5Pattern;
-import org.apache.oro.text.regex.PatternMatcher;
-import org.apache.oro.text.regex.MatchResult;
-import org.apache.oro.text.regex.MalformedPatternException;
 import org.apache.tika.mime.MimeType;
 
 import org.slf4j.Logger;
@@ -51,9 +44,10 @@ import java.text.ParseException;
 import java.text.SimpleDateFormat;
 
 import java.util.Date;
+import java.util.regex.*;
 
-import org.apache.commons.lang.time.DateUtils;
 
+import org.apache.commons.lang.time.DateUtils;
 
 /**
  * Add (or reset) a few metaData properties as respective fields
@@ -75,8 +69,8 @@ public class MoreIndexingFilter implemen
   private boolean MAGIC;
 
   /** Get the MimeTypes resolver instance. */
-  private MimeUtil MIME; 
-  
+  private MimeUtil MIME;
+
   public NutchDocument filter(NutchDocument doc, Parse parse, Text url, 
CrawlDatum datum, Inlinks inlinks)
     throws IndexingException {
 
@@ -89,7 +83,7 @@ public class MoreIndexingFilter implemen
 
     return doc;
   }
-    
+
   // Add time related meta info.  Add last-modified if present.  Index date as
   // last-modified, or, if that's not present, use fetch time.
   private NutchDocument addTime(NutchDocument doc, ParseData data,
@@ -118,43 +112,43 @@ public class MoreIndexingFilter implemen
     try {
       time = HttpDateFormat.toLong(date);
     } catch (ParseException e) {
-       // try to parse it as date in alternative format
-       try {
-           Date parsedDate = DateUtils.parseDate(date,
-                 new String [] {
-                     "EEE MMM dd HH:mm:ss yyyy",
-                     "EEE MMM dd HH:mm:ss yyyy zzz",
-                     "EEE MMM dd HH:mm:ss zzz yyyy",
-                     "EEE, MMM dd HH:mm:ss yyyy zzz",
-                     "EEE, dd MMM yyyy HH:mm:ss zzz",
-                     "EEE,dd MMM yyyy HH:mm:ss zzz",
-                     "EEE, dd MMM yyyy HH:mm:sszzz",
-                     "EEE, dd MMM yyyy HH:mm:ss",
-                     "EEE, dd-MMM-yy HH:mm:ss zzz",
-                     "yyyy/MM/dd HH:mm:ss.SSS zzz",
-                     "yyyy/MM/dd HH:mm:ss.SSS",
-                     "yyyy/MM/dd HH:mm:ss zzz",
-                     "yyyy/MM/dd",
-                     "yyyy.MM.dd HH:mm:ss",
-                     "yyyy-MM-dd HH:mm",
-                     "MMM dd yyyy HH:mm:ss. zzz",
-                     "MMM dd yyyy HH:mm:ss zzz",
-                     "dd.MM.yyyy HH:mm:ss zzz",
-                     "dd MM yyyy HH:mm:ss zzz",
-                     "dd.MM.yyyy; HH:mm:ss",
-                     "dd.MM.yyyy HH:mm:ss",
-                     "dd.MM.yyyy zzz",
-                     "yyyy-MM-dd'T'HH:mm:ss'Z'"
-                 });
-           time = parsedDate.getTime();
+  // try to parse it as date in alternative format
+  try {
+      Date parsedDate = DateUtils.parseDate(date,
+      new String [] {
+          "EEE MMM dd HH:mm:ss yyyy",
+          "EEE MMM dd HH:mm:ss yyyy zzz",
+          "EEE MMM dd HH:mm:ss zzz yyyy",
+          "EEE, MMM dd HH:mm:ss yyyy zzz",
+          "EEE, dd MMM yyyy HH:mm:ss zzz",
+          "EEE,dd MMM yyyy HH:mm:ss zzz",
+          "EEE, dd MMM yyyy HH:mm:sszzz",
+          "EEE, dd MMM yyyy HH:mm:ss",
+          "EEE, dd-MMM-yy HH:mm:ss zzz",
+          "yyyy/MM/dd HH:mm:ss.SSS zzz",
+          "yyyy/MM/dd HH:mm:ss.SSS",
+          "yyyy/MM/dd HH:mm:ss zzz",
+          "yyyy/MM/dd",
+          "yyyy.MM.dd HH:mm:ss",
+          "yyyy-MM-dd HH:mm",
+          "MMM dd yyyy HH:mm:ss. zzz",
+          "MMM dd yyyy HH:mm:ss zzz",
+          "dd.MM.yyyy HH:mm:ss zzz",
+          "dd MM yyyy HH:mm:ss zzz",
+          "dd.MM.yyyy; HH:mm:ss",
+          "dd.MM.yyyy HH:mm:ss",
+          "dd.MM.yyyy zzz",
+          "yyyy-MM-dd'T'HH:mm:ss'Z'"
+      });
+      time = parsedDate.getTime();
             // if (LOG.isWarnEnabled()) {
-           //   LOG.warn(url + ": parsed date: " + date +" to:"+time);
+      //   LOG.warn(url + ": parsed date: " + date +" to:"+time);
             // }
-       } catch (Exception e2) {
+  } catch (Exception e2) {
             if (LOG.isWarnEnabled()) {
-             LOG.warn(url + ": can't parse erroneous date: " + date);
+        LOG.warn(url + ": can't parse erroneous date: " + date);
             }
-       }
+  }
     }
     return time;
   }
@@ -184,7 +178,7 @@ public class MoreIndexingFilter implemen
    * all case insensitive. The query filter is implemented in
    * {@link TypeQueryFilter}.
    * </p>
-   * 
+   *
    * @param doc
    * @param data
    * @param url
@@ -209,14 +203,14 @@ public class MoreIndexingFilter implemen
     } else {
       mimeType = MIME.forName(MimeUtil.cleanMimeType(contentType));
     }
-        
+
     // Checks if we solved the content-type.
     if (mimeType == null) {
       return doc;
     }
 
     contentType = mimeType.getName();
-    
+
     doc.add("type", contentType);
 
     // Check if we need to split the content type in sub parts
@@ -227,14 +221,14 @@ public class MoreIndexingFilter implemen
         doc.add("type", part);
       }
     }
-    
+
     // leave this for future improvement
     //MimeTypeParameterList parameterList = mimeType.getParameters()
 
     return doc;
   }
 
-  
+
   /**
    * Utility method for splitting mime type into type and subtype.
    * @param mimeType
@@ -251,19 +245,18 @@ public class MoreIndexingFilter implemen
   // Patterns used to extract filename from possible non-standard
   // HTTP header "Content-Disposition". Typically it looks like:
   // Content-Disposition: inline; filename="foo.ppt"
-  private PatternMatcher matcher = new Perl5Matcher();
-
   private Configuration conf;
-  static Perl5Pattern patterns[] = {null, null};
+
+  static Pattern patterns[] = {null, null};
+
   static {
-    Perl5Compiler compiler = new Perl5Compiler();
     try {
       // order here is important
       patterns[0] =
-        (Perl5Pattern) compiler.compile("\\bfilename=['\"](.+)['\"]");
+        Pattern.compile("\\bfilename=['\"](.+)['\"]");
       patterns[1] =
-        (Perl5Pattern) compiler.compile("\\bfilename=(\\S+)\\b");
-    } catch (MalformedPatternException e) {
+        Pattern.compile("\\bfilename=(\\S+)\\b");
+    } catch (PatternSyntaxException e) {
       // just ignore
     }
   }
@@ -273,11 +266,10 @@ public class MoreIndexingFilter implemen
     if (contentDisposition == null)
       return doc;
 
-    MatchResult result;
     for (int i=0; i<patterns.length; i++) {
-      if (matcher.contains(contentDisposition,patterns[i])) {
-        result = matcher.getMatch();
-        doc.add("title", result.group(1));
+      Matcher matcher = patterns[i].matcher(contentDisposition);
+      if (matcher.find()) {
+        doc.add("title", matcher.group(1));
         break;
       }
     }

Modified: 
nutch/trunk/src/plugin/index-more/src/test/org/apache/nutch/indexer/more/TestMoreIndexingFilter.java
URL: 
http://svn.apache.org/viewvc/nutch/trunk/src/plugin/index-more/src/test/org/apache/nutch/indexer/more/TestMoreIndexingFilter.java?rev=1200360&r1=1200359&r2=1200360&view=diff
==============================================================================
--- 
nutch/trunk/src/plugin/index-more/src/test/org/apache/nutch/indexer/more/TestMoreIndexingFilter.java
 (original)
+++ 
nutch/trunk/src/plugin/index-more/src/test/org/apache/nutch/indexer/more/TestMoreIndexingFilter.java
 Thu Nov 10 15:02:04 2011
@@ -70,6 +70,21 @@ public class TestMoreIndexingFilter exte
     assertEquals("text/html", doc.getFieldValue("type"));    
   }
 
+  public void testContentDispositionTitle() throws IndexingException {
+    Configuration conf = NutchConfiguration.create();
+
+    Metadata metadata = new Metadata();
+    metadata.add(Response.CONTENT_DISPOSITION, "filename=filename.ext");
+    MoreIndexingFilter filter = new MoreIndexingFilter();
+    filter.setConf(conf);
+
+    NutchDocument doc = filter.filter(new NutchDocument(), new 
ParseImpl("text", new ParseData(
+      new ParseStatus(), "title", new Outlink[0], metadata)), new Text(
+        "http://www.example.com/";), new CrawlDatum(), new Inlinks());
+
+    assertEquals("content-disposition not detected", "filename.ext", 
doc.getFieldValue("title"));
+  }
+
   private void assertParts(String[] parts, int count, String... expected) {
     assertEquals(count, parts.length);
     for (int i = 0; i < expected.length; i++) {


Reply via email to