Author: markus
Date: Thu Nov 10 15:02:04 2011
New Revision: 1200360
URL: http://svn.apache.org/viewvc?rev=1200360&view=rev
Log:
NUTCH-1061 Migrate MoreIndexingFilter from Apache ORO to java.util.regex
Modified:
nutch/trunk/CHANGES.txt
nutch/trunk/src/plugin/index-more/plugin.xml
nutch/trunk/src/plugin/index-more/src/java/org/apache/nutch/indexer/more/MoreIndexingFilter.java
nutch/trunk/src/plugin/index-more/src/test/org/apache/nutch/indexer/more/TestMoreIndexingFilter.java
Modified: nutch/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/nutch/trunk/CHANGES.txt?rev=1200360&r1=1200359&r2=1200360&view=diff
==============================================================================
--- nutch/trunk/CHANGES.txt (original)
+++ nutch/trunk/CHANGES.txt Thu Nov 10 15:02:04 2011
@@ -1,5 +1,7 @@
Nutch Change Log
+* NUTCH-1061 Migrate MoreIndexingFilter from Apache ORO to java.util.regex
(markus)
+
* NUTCH-1178 Incorrect CSV header CrawlDatumCsvOutputFormat (markus)
* NUTCH-1142 Normalization and filtering in WebGraph (markus)
Modified: nutch/trunk/src/plugin/index-more/plugin.xml
URL:
http://svn.apache.org/viewvc/nutch/trunk/src/plugin/index-more/plugin.xml?rev=1200360&r1=1200359&r2=1200360&view=diff
==============================================================================
--- nutch/trunk/src/plugin/index-more/plugin.xml (original)
+++ nutch/trunk/src/plugin/index-more/plugin.xml Thu Nov 10 15:02:04 2011
@@ -39,14 +39,4 @@
class="org.apache.nutch.indexer.more.MoreIndexingFilter"/>
</extension>
- <!--
- <extension id="org.apache.nutch.searcher.more.MoreQueryFilter"
- name="Nutch More Query Filter"
- point="org.apache.nutch.searcher.QueryFilter">
- <implementation id="MoreQueryFilter"
- class="org.apache.nutch.searcher.more.MoreQueryFilter"
- raw-fields="more"/>
- </extension>
- -->
-
</plugin>
Modified:
nutch/trunk/src/plugin/index-more/src/java/org/apache/nutch/indexer/more/MoreIndexingFilter.java
URL:
http://svn.apache.org/viewvc/nutch/trunk/src/plugin/index-more/src/java/org/apache/nutch/indexer/more/MoreIndexingFilter.java?rev=1200360&r1=1200359&r2=1200360&view=diff
==============================================================================
---
nutch/trunk/src/plugin/index-more/src/java/org/apache/nutch/indexer/more/MoreIndexingFilter.java
(original)
+++
nutch/trunk/src/plugin/index-more/src/java/org/apache/nutch/indexer/more/MoreIndexingFilter.java
Thu Nov 10 15:02:04 2011
@@ -16,13 +16,6 @@
*/
package org.apache.nutch.indexer.more;
-
-import org.apache.oro.text.regex.Perl5Compiler;
-import org.apache.oro.text.regex.Perl5Matcher;
-import org.apache.oro.text.regex.Perl5Pattern;
-import org.apache.oro.text.regex.PatternMatcher;
-import org.apache.oro.text.regex.MatchResult;
-import org.apache.oro.text.regex.MalformedPatternException;
import org.apache.tika.mime.MimeType;
import org.slf4j.Logger;
@@ -51,9 +44,10 @@ import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Date;
+import java.util.regex.*;
-import org.apache.commons.lang.time.DateUtils;
+import org.apache.commons.lang.time.DateUtils;
/**
* Add (or reset) a few metaData properties as respective fields
@@ -75,8 +69,8 @@ public class MoreIndexingFilter implemen
private boolean MAGIC;
/** Get the MimeTypes resolver instance. */
- private MimeUtil MIME;
-
+ private MimeUtil MIME;
+
public NutchDocument filter(NutchDocument doc, Parse parse, Text url,
CrawlDatum datum, Inlinks inlinks)
throws IndexingException {
@@ -89,7 +83,7 @@ public class MoreIndexingFilter implemen
return doc;
}
-
+
// Add time related meta info. Add last-modified if present. Index date as
// last-modified, or, if that's not present, use fetch time.
private NutchDocument addTime(NutchDocument doc, ParseData data,
@@ -118,43 +112,43 @@ public class MoreIndexingFilter implemen
try {
time = HttpDateFormat.toLong(date);
} catch (ParseException e) {
- // try to parse it as date in alternative format
- try {
- Date parsedDate = DateUtils.parseDate(date,
- new String [] {
- "EEE MMM dd HH:mm:ss yyyy",
- "EEE MMM dd HH:mm:ss yyyy zzz",
- "EEE MMM dd HH:mm:ss zzz yyyy",
- "EEE, MMM dd HH:mm:ss yyyy zzz",
- "EEE, dd MMM yyyy HH:mm:ss zzz",
- "EEE,dd MMM yyyy HH:mm:ss zzz",
- "EEE, dd MMM yyyy HH:mm:sszzz",
- "EEE, dd MMM yyyy HH:mm:ss",
- "EEE, dd-MMM-yy HH:mm:ss zzz",
- "yyyy/MM/dd HH:mm:ss.SSS zzz",
- "yyyy/MM/dd HH:mm:ss.SSS",
- "yyyy/MM/dd HH:mm:ss zzz",
- "yyyy/MM/dd",
- "yyyy.MM.dd HH:mm:ss",
- "yyyy-MM-dd HH:mm",
- "MMM dd yyyy HH:mm:ss. zzz",
- "MMM dd yyyy HH:mm:ss zzz",
- "dd.MM.yyyy HH:mm:ss zzz",
- "dd MM yyyy HH:mm:ss zzz",
- "dd.MM.yyyy; HH:mm:ss",
- "dd.MM.yyyy HH:mm:ss",
- "dd.MM.yyyy zzz",
- "yyyy-MM-dd'T'HH:mm:ss'Z'"
- });
- time = parsedDate.getTime();
+ // try to parse it as date in alternative format
+ try {
+ Date parsedDate = DateUtils.parseDate(date,
+ new String [] {
+ "EEE MMM dd HH:mm:ss yyyy",
+ "EEE MMM dd HH:mm:ss yyyy zzz",
+ "EEE MMM dd HH:mm:ss zzz yyyy",
+ "EEE, MMM dd HH:mm:ss yyyy zzz",
+ "EEE, dd MMM yyyy HH:mm:ss zzz",
+ "EEE,dd MMM yyyy HH:mm:ss zzz",
+ "EEE, dd MMM yyyy HH:mm:sszzz",
+ "EEE, dd MMM yyyy HH:mm:ss",
+ "EEE, dd-MMM-yy HH:mm:ss zzz",
+ "yyyy/MM/dd HH:mm:ss.SSS zzz",
+ "yyyy/MM/dd HH:mm:ss.SSS",
+ "yyyy/MM/dd HH:mm:ss zzz",
+ "yyyy/MM/dd",
+ "yyyy.MM.dd HH:mm:ss",
+ "yyyy-MM-dd HH:mm",
+ "MMM dd yyyy HH:mm:ss. zzz",
+ "MMM dd yyyy HH:mm:ss zzz",
+ "dd.MM.yyyy HH:mm:ss zzz",
+ "dd MM yyyy HH:mm:ss zzz",
+ "dd.MM.yyyy; HH:mm:ss",
+ "dd.MM.yyyy HH:mm:ss",
+ "dd.MM.yyyy zzz",
+ "yyyy-MM-dd'T'HH:mm:ss'Z'"
+ });
+ time = parsedDate.getTime();
// if (LOG.isWarnEnabled()) {
- // LOG.warn(url + ": parsed date: " + date +" to:"+time);
+ // LOG.warn(url + ": parsed date: " + date +" to:"+time);
// }
- } catch (Exception e2) {
+ } catch (Exception e2) {
if (LOG.isWarnEnabled()) {
- LOG.warn(url + ": can't parse erroneous date: " + date);
+ LOG.warn(url + ": can't parse erroneous date: " + date);
}
- }
+ }
}
return time;
}
@@ -184,7 +178,7 @@ public class MoreIndexingFilter implemen
* all case insensitive. The query filter is implemented in
* {@link TypeQueryFilter}.
* </p>
- *
+ *
* @param doc
* @param data
* @param url
@@ -209,14 +203,14 @@ public class MoreIndexingFilter implemen
} else {
mimeType = MIME.forName(MimeUtil.cleanMimeType(contentType));
}
-
+
// Checks if we solved the content-type.
if (mimeType == null) {
return doc;
}
contentType = mimeType.getName();
-
+
doc.add("type", contentType);
// Check if we need to split the content type in sub parts
@@ -227,14 +221,14 @@ public class MoreIndexingFilter implemen
doc.add("type", part);
}
}
-
+
// leave this for future improvement
//MimeTypeParameterList parameterList = mimeType.getParameters()
return doc;
}
-
+
/**
* Utility method for splitting mime type into type and subtype.
* @param mimeType
@@ -251,19 +245,18 @@ public class MoreIndexingFilter implemen
// Patterns used to extract filename from possible non-standard
// HTTP header "Content-Disposition". Typically it looks like:
// Content-Disposition: inline; filename="foo.ppt"
- private PatternMatcher matcher = new Perl5Matcher();
-
private Configuration conf;
- static Perl5Pattern patterns[] = {null, null};
+
+ static Pattern patterns[] = {null, null};
+
static {
- Perl5Compiler compiler = new Perl5Compiler();
try {
// order here is important
patterns[0] =
- (Perl5Pattern) compiler.compile("\\bfilename=['\"](.+)['\"]");
+ Pattern.compile("\\bfilename=['\"](.+)['\"]");
patterns[1] =
- (Perl5Pattern) compiler.compile("\\bfilename=(\\S+)\\b");
- } catch (MalformedPatternException e) {
+ Pattern.compile("\\bfilename=(\\S+)\\b");
+ } catch (PatternSyntaxException e) {
// just ignore
}
}
@@ -273,11 +266,10 @@ public class MoreIndexingFilter implemen
if (contentDisposition == null)
return doc;
- MatchResult result;
for (int i=0; i<patterns.length; i++) {
- if (matcher.contains(contentDisposition,patterns[i])) {
- result = matcher.getMatch();
- doc.add("title", result.group(1));
+ Matcher matcher = patterns[i].matcher(contentDisposition);
+ if (matcher.find()) {
+ doc.add("title", matcher.group(1));
break;
}
}
Modified:
nutch/trunk/src/plugin/index-more/src/test/org/apache/nutch/indexer/more/TestMoreIndexingFilter.java
URL:
http://svn.apache.org/viewvc/nutch/trunk/src/plugin/index-more/src/test/org/apache/nutch/indexer/more/TestMoreIndexingFilter.java?rev=1200360&r1=1200359&r2=1200360&view=diff
==============================================================================
---
nutch/trunk/src/plugin/index-more/src/test/org/apache/nutch/indexer/more/TestMoreIndexingFilter.java
(original)
+++
nutch/trunk/src/plugin/index-more/src/test/org/apache/nutch/indexer/more/TestMoreIndexingFilter.java
Thu Nov 10 15:02:04 2011
@@ -70,6 +70,21 @@ public class TestMoreIndexingFilter exte
assertEquals("text/html", doc.getFieldValue("type"));
}
+ public void testContentDispositionTitle() throws IndexingException {
+ Configuration conf = NutchConfiguration.create();
+
+ Metadata metadata = new Metadata();
+ metadata.add(Response.CONTENT_DISPOSITION, "filename=filename.ext");
+ MoreIndexingFilter filter = new MoreIndexingFilter();
+ filter.setConf(conf);
+
+ NutchDocument doc = filter.filter(new NutchDocument(), new
ParseImpl("text", new ParseData(
+ new ParseStatus(), "title", new Outlink[0], metadata)), new Text(
+ "http://www.example.com/"), new CrawlDatum(), new Inlinks());
+
+ assertEquals("content-disposition not detected", "filename.ext",
doc.getFieldValue("title"));
+ }
+
private void assertParts(String[] parts, int count, String... expected) {
assertEquals(count, parts.length);
for (int i = 0; i < expected.length; i++) {