Revision: 16591
http://sourceforge.net/p/gate/code/16591
Author: markagreenwood
Date: 2013-03-13 12:44:01 +0000 (Wed, 13 Mar 2013)
Log Message:
-----------
fixed a bug that wouldn't allow dates that started with numbers and suffixes
(like 31st), refactored to make it easier to provide otherways of finding dates
but use the same normalization code, andd added such a PR that can normalise
the ANNIE date annotations
Modified Paths:
--------------
gate/trunk/plugins/Tagger_DateNormalizer/src/gate/creole/dates/DateNormalizer.java
Added Paths:
-----------
gate/trunk/plugins/Tagger_DateNormalizer/src/gate/creole/dates/DateAnnotationNormalizer.java
Added:
gate/trunk/plugins/Tagger_DateNormalizer/src/gate/creole/dates/DateAnnotationNormalizer.java
===================================================================
---
gate/trunk/plugins/Tagger_DateNormalizer/src/gate/creole/dates/DateAnnotationNormalizer.java
(rev 0)
+++
gate/trunk/plugins/Tagger_DateNormalizer/src/gate/creole/dates/DateAnnotationNormalizer.java
2013-03-13 12:44:01 UTC (rev 16591)
@@ -0,0 +1,120 @@
+/*
+ * DateAnnotationNormalizer.java
+ *
+ * Copyright (c) 2013, The University of Sheffield.
+ *
+ * This file is part of GATE (see http://gate.ac.uk/), and is free software,
+ * licenced under the GNU Library General Public License, Version 3, June 2007
+ * (in the distribution as file licence.html, and also available at
+ * http://gate.ac.uk/gate/licence.html).
+ *
+ * Mark A. Greenwood, 13/03/2013
+ */
+package gate.creole.dates;
+
+import gate.Annotation;
+import gate.AnnotationSet;
+import gate.FeatureMap;
+import gate.Utils;
+import gate.creole.ExecutionException;
+import gate.creole.ExecutionInterruptedException;
+import gate.creole.metadata.CreoleParameter;
+import gate.creole.metadata.CreoleResource;
+import gate.creole.metadata.RunTime;
+
+import java.text.DateFormat;
+import java.util.Date;
+import java.util.regex.Pattern;
+
+import mark.util.DateParser;
+import mark.util.ParsePositionEx;
+
+@CreoleResource(name = "Date Annotation Normalizer", icon =
"date-normalizer.png", comment = "provides normalized values for all existing
date annotations")
+public class DateAnnotationNormalizer extends DateNormalizer {
+
+ private String annotationFeature;
+
+ private Boolean wholeMatchOnly;
+
+ @RunTime
+ @CreoleParameter(defaultValue = "string", comment = "the annotation feature
to normalize")
+ public void setAnnotationFeature(String name) {
+ annotationFeature = name;
+ }
+
+ public String getAnnotationFeature() {
+ return annotationFeature;
+ }
+
+ @RunTime
+ @CreoleParameter(defaultValue = "true", comment = "only normalize if the
whole feature is parsed")
+ public void setWholeMatchOnly(Boolean wholeMatchOnly) {
+ this.wholeMatchOnly = wholeMatchOnly;
+ }
+
+ public Boolean getWholeMatchOnly() {
+ return wholeMatchOnly;
+ }
+
+ @Override
+ protected void annotate(Date documentDate, DateParser dp, DateFormat df)
throws ExecutionException {
+ AnnotationSet dates =
document.getAnnotations(getInputASName()).get(getAnnotationName());
+
+ ParsePositionEx pp = new ParsePositionEx();
+
+ float progress = 0;
+ for (Annotation date : dates) {
+
+ // if we have been asked to stop then do so
+ if(isInterrupted()) { throw new ExecutionInterruptedException(
+ "The execution of the \"" + getName()
+ + "\" Date Normalizer has been abruptly interrupted!"); }
+
+ FeatureMap params = date.getFeatures();
+
+ String text = (String)params.get(annotationFeature);
+
+ if (text == null && annotationFeature.equals("string")) text =
Utils.stringFor(document, date);
+
+ try {
+ // try and parse the document content starting from the beginning of
the
+ // current token
+ Date d =
+ dp.parse(text, pp.reset(0), documentDate);
+
+ if(d == null) {
+ //if the text didn't parse skip on to the next character and try
again
+ //start++;
+ continue;
+ }
+
+ if (wholeMatchOnly && pp.getIndex() != text.length()) {
+ continue;
+ }
+
+ // normalize the date and store the value
+ params.put("normalized", getNumericOutput()
+ ? Integer.parseInt(df.format(d))
+ : df.format(d));
+ // set the complete feature based on the inferred flags from the parser
+ if(pp.getFeatures().get("inferred").equals(DateParser.NONE)) {
+ params.put("complete", "true");
+ } else {
+ params.put("complete", "false");
+ }
+
+ //store the inferred flags from the parser so people can have fine
+ //grained control if they need it
+ params.put("inferred", pp.getFeatures().get("inferred"));
+
+ // copy the relative date feature from the parser into the feature map
+ params.put("relative", pp.getFeatures().get("relative"));
+
+
+ }
+ finally {
+ fireProgressChanged((int)((++progress / dates.size()) * 100));
+ }
+ }
+ }
+}
Modified:
gate/trunk/plugins/Tagger_DateNormalizer/src/gate/creole/dates/DateNormalizer.java
===================================================================
---
gate/trunk/plugins/Tagger_DateNormalizer/src/gate/creole/dates/DateNormalizer.java
2013-03-13 11:02:43 UTC (rev 16590)
+++
gate/trunk/plugins/Tagger_DateNormalizer/src/gate/creole/dates/DateNormalizer.java
2013-03-13 12:44:01 UTC (rev 16591)
@@ -1,7 +1,7 @@
/*
- * Normalizer.java
+ * DateNormalizer.java
*
- * Copyright (c) 2010-2011, The University of Sheffield.
+ * Copyright (c) 2010-2013, The University of Sheffield.
*
* This file is part of GATE (see http://gate.ac.uk/), and is free software,
* licenced under the GNU Library General Public License, Version 3, June 2007
@@ -17,6 +17,7 @@
import gate.Factory;
import gate.FeatureMap;
import gate.Resource;
+import gate.Utils;
import gate.creole.AbstractLanguageAnalyser;
import gate.creole.ExecutionException;
import gate.creole.ExecutionInterruptedException;
@@ -41,8 +42,6 @@
import mark.util.DateParser;
import mark.util.ParsePositionEx;
-import org.apache.log4j.Logger;
-
/**
* A GATE PR which attempts to normalise dates within a document against the
* date at which the document was written or published. This PR wraps the open
@@ -58,8 +57,6 @@
public class DateNormalizer extends AbstractLanguageAnalyser {
private static final long serialVersionUID = -6580533128028166284L;
- private transient Logger logger =
Logger.getLogger(this.getClass().getName());
-
/**
* A comparator that orders annotations by priority (an Integer annotation
* feature) and then by offset
@@ -246,22 +243,15 @@
DateParser.getLocale((String)document.getFeatures().get("locale"));
if(docLocale == null) docLocale = locale;
if(docLocale == null) docLocale = Locale.getDefault();
+
// create an instance of the parser
+ //TODO cache the parser for the init time locale
DateParser dp = new DateParser(docLocale);
- //now we have a parser create a regexp to look for possible dates
- StringBuilder pattern = new StringBuilder("\\b([0-9]{1,4}");
- for (String word : dp.getWords()) {
- if (word.length() > 0) pattern.append("|(").append(word).append(")");
- }
- pattern.append(")\\b");
- Pattern finder =
Pattern.compile(pattern.toString(),Pattern.CASE_INSENSITIVE);
-
// get handles to the document content and the input and output annotation
// sets so that we can easily refer to them later
- String docContent = document.getContent().toString();
AnnotationSet inputAS = document.getAnnotations(inputASName);
- AnnotationSet outputAS = document.getAnnotations(outputASName);
+
// a parse position that will help us to parse the document dates
ParsePositionEx pp = new ParsePositionEx();
// lets try and figure out what date the document was written on
@@ -287,8 +277,7 @@
// either from the specified feature or from the underlying
string
if(parts.length == 1) {
dd =
- docContent.substring(a.getStartNode().getOffset()
- .intValue(), a.getEndNode().getOffset().intValue());
+ Utils.stringFor(document, a);
} else {
dd = (String)a.getFeatures().get(parts[1]);
}
@@ -333,9 +322,41 @@
.format(documentDate)));
}
+ annotate(documentDate, dp, df);
+
+ // we have finished so update anyone who cares
+ fireProcessFinished();
+ fireStatusChanged("Dates detected and normalized in \""
+ + document.getName()
+ + "\" in "
+ + NumberFormat.getInstance().format(
+ (double)(System.currentTimeMillis() - startTime) / 1000)
+ + " seconds!");
+ }
+
+ /**
+ * This method actually does all the work, but it is here so it can be
+ * overriden by subclasses to do things differently
+ */
+ protected void annotate(Date documentDate, DateParser dp, DateFormat df)
throws ExecutionException {
+
+ //now we have a parser create a regexp to look for possible dates
+ StringBuilder pattern = new StringBuilder("\\b([0-9]{1,4}");
+ for (String word : dp.getWords()) {
+ if (word.length() > 0) pattern.append("|(").append(word).append(")");
+ }
+ pattern.append(")");
+ Pattern finder =
Pattern.compile(pattern.toString(),Pattern.CASE_INSENSITIVE);
+
+ String docContent = document.getContent().toString();
+
//get a matcher for possible dates over the content
Matcher m = finder.matcher(docContent);
+ ParsePositionEx pp = new ParsePositionEx();
+
+ AnnotationSet outputAS = document.getAnnotations(outputASName);
+
int start = 0;
while (m.find()) {
@@ -390,18 +411,9 @@
// the next character in the document
start++;
}
- // calcualte percentage complete using the parsing position within the
+ // Calculate percentage complete using the parsing position within the
// document content
- fireProgressChanged((int)(start / docContent.length()) * 100);
+ fireProgressChanged((int)(((float)start / docContent.length()) * 100));
}
-
- // we have finished so update anyone who cares
- fireProcessFinished();
- fireStatusChanged("Dates detected and normalized in \""
- + document.getName()
- + "\" in "
- + NumberFormat.getInstance().format(
- (double)(System.currentTimeMillis() - startTime) / 1000)
- + " seconds!");
}
}
This was sent by the SourceForge.net collaborative development platform, the
world's largest Open Source development site.
------------------------------------------------------------------------------
Everyone hates slow websites. So do we.
Make your web apps faster with AppDynamics
Download AppDynamics Lite for free today:
http://p.sf.net/sfu/appdyn_d2d_mar
_______________________________________________
GATE-cvs mailing list
[email protected]
https://lists.sourceforge.net/lists/listinfo/gate-cvs
