Revision: 16275
http://gate.svn.sourceforge.net/gate/?rev=16275&view=rev
Author: adamfunk
Date: 2012-11-14 12:52:58 +0000 (Wed, 14 Nov 2012)
Log Message:
-----------
Now creating Split annotations, mostly the same as ANNIE.
Normalized the use of ANNIEConstants.
Modified Paths:
--------------
gate/trunk/plugins/OpenNLP/src/gate/opennlp/OpenNLPNameFin.java
gate/trunk/plugins/OpenNLP/src/gate/opennlp/OpenNlpChunker.java
gate/trunk/plugins/OpenNLP/src/gate/opennlp/OpenNlpPOS.java
gate/trunk/plugins/OpenNLP/src/gate/opennlp/OpenNlpSentenceSplit.java
gate/trunk/plugins/OpenNLP/src/gate/opennlp/OpenNlpTokenizer.java
gate/trunk/plugins/OpenNLP/src/gate/opennlp/Sentence.java
Modified: gate/trunk/plugins/OpenNLP/src/gate/opennlp/OpenNLPNameFin.java
===================================================================
--- gate/trunk/plugins/OpenNLP/src/gate/opennlp/OpenNLPNameFin.java
2012-11-14 02:22:00 UTC (rev 16274)
+++ gate/trunk/plugins/OpenNLP/src/gate/opennlp/OpenNLPNameFin.java
2012-11-14 12:52:58 UTC (rev 16275)
@@ -35,24 +35,19 @@
comment = "NER PR using a set of OpenNLP maxent models",
helpURL =
"http://gate.ac.uk/sale/tao/splitch21.html#sec:misc-creole:opennlp";)
public class OpenNLPNameFin extends AbstractLanguageAnalyser {
-
+
private static final long serialVersionUID = -5507338627058320125L;
private static final Logger logger = Logger.getLogger(OpenNLPNameFin.class);
-
-
- /* CREOLE PARAMETERS & SUCH*/
+
+
+ /* CREOLE PARAMETERS & WRAPPED COMPONENTS */
private String inputASName, outputASName;
private URL configUrl;
private Map<NameFinderME, String> finders;
- private String tokenType = ANNIEConstants.TOKEN_ANNOTATION_TYPE;
- private String sentenceType = ANNIEConstants.SENTENCE_ANNOTATION_TYPE;
- private String posFeature = ANNIEConstants.TOKEN_CATEGORY_FEATURE_NAME;
- private String stringFeature = ANNIEConstants.TOKEN_STRING_FEATURE_NAME;
-
- @Override
- public void execute() throws ExecutionException {
+ @Override
+ public void execute() throws ExecutionException {
interrupted = false;
long startTime = System.currentTimeMillis();
if(document == null) {
@@ -61,111 +56,117 @@
fireStatusChanged("Running " + this.getName() + " on " +
document.getName());
fireProgressChanged(0);
- AnnotationSet inputAS = document.getAnnotations(inputASName);
- AnnotationSet outputAS = document.getAnnotations(outputASName);
- AnnotationSet sentences = inputAS.get(sentenceType);
+ AnnotationSet inputAS = document.getAnnotations(inputASName);
+ AnnotationSet outputAS = document.getAnnotations(outputASName);
+ AnnotationSet sentences = inputAS.get(SENTENCE_ANNOTATION_TYPE);
- int nbrDone = 0;
- int nbrSentences = sentences.size();
-
- for (Annotation sentence : sentences) {
- /* For each input Sentence annotation, produce a list of
- * Token.string values and the data structure for translating
- * offsets. */
- AnnotationSet tokenSet =
Utils.getContainedAnnotations(inputAS, sentence, tokenType);
- Sentence tokens = new Sentence(tokenSet, stringFeature,
posFeature);
- String[] strings = tokens.getStrings();
-
- // Run each NameFinder over the sentence
- for (NameFinderME finder : finders.keySet()) {
- String type = finders.get(finder);
- Span[] spans = finder.find(strings);
+ int nbrDone = 0;
+ int nbrSentences = sentences.size();
- for (Span span : spans) {
- // Translate the offsets and create the output NE
annotation
- long start = tokens.getStartOffset(span);
- long end = tokens.getEndOffset(span);
- FeatureMap fm = Factory.newFeatureMap();
- fm.put("source", "OpenNLP");
- try {
- outputAS.add(start, end, type, fm);
- }
- catch (InvalidOffsetException e) {
- throw new ExecutionException(e);
- }
-
- if(isInterrupted()) {
- throw new ExecutionInterruptedException("Execution of "
+
- this.getName() + " has been abruptly interrupted!");
- }
- } // end loop over names from 1 finder in 1 sentence
- } // end loop over NameFinders within one sentence
+ for (Annotation sentence : sentences) {
+ /* For each input Sentence annotation, produce a list of
+ * Token.string values and the data structure for translating
+ * offsets. */
+ AnnotationSet tokenSet = Utils.getContainedAnnotations(inputAS,
sentence, TOKEN_ANNOTATION_TYPE);
+ Sentence tokens = new Sentence(tokenSet, TOKEN_STRING_FEATURE_NAME,
TOKEN_CATEGORY_FEATURE_NAME);
+ String[] strings = tokens.getStrings();
+
+ // Run each NameFinder over the sentence
+ for (NameFinderME finder : finders.keySet()) {
+ String type = finders.get(finder);
+ Span[] spans = finder.find(strings);
+
+ for (Span span : spans) {
+ // Translate the offsets and create the output NE annotation
+ long start = tokens.getStartOffset(span);
+ long end = tokens.getEndOffset(span);
+ FeatureMap fm = Factory.newFeatureMap();
+ fm.put("source", "OpenNLP");
+ try {
+ outputAS.add(start, end, type, fm);
+ }
+ catch (InvalidOffsetException e) {
+ throw new ExecutionException(e);
+ }
+
+ if(isInterrupted()) {
+ throw new ExecutionInterruptedException("Execution of " +
+ this.getName() + " has been abruptly interrupted!");
+ }
+ } // end loop over names from 1 finder in 1 sentence
+ } // end loop over NameFinders within one sentence
nbrDone++;
fireProgressChanged((int)(100 * nbrDone / nbrSentences));
- } // end for sentence : sentences
-
+ } // end for sentence : sentences
+
fireProcessFinished();
fireStatusChanged("Finished " + this.getName() + " on " +
document.getName()
+ " in " + NumberFormat.getInstance().format(
(double)(System.currentTimeMillis() - startTime) / 1000)
- + " seconds!");
- }
+ + " seconds!");
+ }
-
- @Override
- public Resource init() throws ResourceInstantiationException {
- try {
- loadModels(this.configUrl);
- }
- catch (IOException e) {
- throw new ResourceInstantiationException(e);
- }
+
+ @Override
+ public Resource init() throws ResourceInstantiationException {
+ try {
+ loadModels(this.configUrl);
+ }
+ catch (IOException e) {
+ throw new ResourceInstantiationException(e);
+ }
super.init();
return this;
- }
+ }
-
- private void loadModels(URL configUrl) throws IOException {
- this.finders = new HashMap<NameFinderME, String>();
- Properties properties = new Properties();
- InputStream configInput = null;
- try {
- configInput = configUrl.openStream();
- properties.load(configInput);
- }
- finally {
- if (configInput != null) {
- configInput.close();
- }
- }
-
- Set<String> modelFiles = properties.stringPropertyNames();
- for (String filename : modelFiles) {
- InputStream modelInput = null;
- try {
- URL modelUrl = new URL(configUrl, filename);
- String type = properties.getProperty(filename);
- modelInput = modelUrl.openStream();
- TokenNameFinderModel model = new TokenNameFinderModel(modelInput);
- NameFinderME finder = new NameFinderME(model);
- this.finders.put(finder, type);
- logger.info("OpenNLP NameFinder: " + modelUrl.toString() + " -> "
+ type);
- }
- finally {
- if (modelInput != null) {
- modelInput.close();
- }
- }
- }
- }
-
-
- @Override
- public void reInit() throws ResourceInstantiationException {
- init();
- }
-
+ private void loadModels(URL configUrl) throws IOException {
+ // Make an empty finder->annotationType table
+ this.finders = new HashMap<NameFinderME, String>();
+
+ // Load the config file (flat table, 2 columns)
+ Properties properties = new Properties();
+ InputStream configInput = null;
+ try {
+ configInput = configUrl.openStream();
+ properties.load(configInput);
+ }
+ finally {
+ if (configInput != null) {
+ configInput.close();
+ }
+ }
+
+ // Go through the config entries
+ Set<String> modelFiles = properties.stringPropertyNames();
+ for (String filename : modelFiles) {
+ InputStream modelInput = null;
+ try {
+ // Initialize a NameFinder with this model
+ URL modelUrl = new URL(configUrl, filename);
+ String type = properties.getProperty(filename);
+ modelInput = modelUrl.openStream();
+ TokenNameFinderModel model = new TokenNameFinderModel(modelInput);
+ NameFinderME finder = new NameFinderME(model);
+ // Add it to the table with its annotation type
+ this.finders.put(finder, type);
+ logger.info("OpenNLP NameFinder: " + modelUrl.toString() + " -> " +
type);
+ }
+ finally {
+ if (modelInput != null) {
+ modelInput.close();
+ }
+ }
+ }
+ }
+
+
+ @Override
+ public void reInit() throws ResourceInstantiationException {
+ init();
+ }
+
+
/* CREOLE PARAMETERS */
@RunTime
@@ -174,7 +175,7 @@
public void setInputASName(String name) {
this.inputASName = name;
}
-
+
public String getInputASName() {
return this.inputASName;
}
@@ -185,7 +186,7 @@
public void setOutputASName(String name) {
this.outputASName = name;
}
-
+
public String getOutputASName() {
return this.outputASName;
}
Modified: gate/trunk/plugins/OpenNLP/src/gate/opennlp/OpenNlpChunker.java
===================================================================
--- gate/trunk/plugins/OpenNLP/src/gate/opennlp/OpenNlpChunker.java
2012-11-14 02:22:00 UTC (rev 16274)
+++ gate/trunk/plugins/OpenNLP/src/gate/opennlp/OpenNlpChunker.java
2012-11-14 12:52:58 UTC (rev 16275)
@@ -31,26 +31,21 @@
@CreoleResource(name = "OpenNLP Chunker",
comment = "Chunker using an OpenNLP maxent model",
helpURL =
"http://gate.ac.uk/sale/tao/splitch21.html#sec:misc-creole:opennlp";)
-public class OpenNlpChunker extends AbstractLanguageAnalyser {
+public class OpenNlpChunker extends AbstractLanguageAnalyser {
private static final long serialVersionUID = 3254481728303447340L;
private static final Logger logger = Logger.getLogger(OpenNlpChunker.class);
-
+
/* CREOLE PARAMETERS & SUCH*/
private String inputASName, outputASName, chunkFeature;
private URL modelUrl;
private ChunkerModel model;
private ChunkerME chunker;
- private String tokenType = ANNIEConstants.TOKEN_ANNOTATION_TYPE;
- private String sentenceType = ANNIEConstants.SENTENCE_ANNOTATION_TYPE;
- private String posFeature = ANNIEConstants.TOKEN_CATEGORY_FEATURE_NAME;
- private String stringFeature = ANNIEConstants.TOKEN_STRING_FEATURE_NAME;
-
- @Override
- public void execute() throws ExecutionException {
+ @Override
+ public void execute() throws ExecutionException {
interrupted = false;
long startTime = System.currentTimeMillis();
if(document == null) {
@@ -59,26 +54,26 @@
fireStatusChanged("Running " + this.getName() + " on " +
document.getName());
fireProgressChanged(0);
-AnnotationSet inputAS = document.getAnnotations(inputASName);
- AnnotationSet outputAS = document.getAnnotations(outputASName);
- boolean sameAS = inputAS.equals(outputAS);
+ AnnotationSet inputAS = document.getAnnotations(inputASName);
+ AnnotationSet outputAS = document.getAnnotations(outputASName);
+ boolean sameAS = inputAS.equals(outputAS);
- AnnotationSet sentences = inputAS.get(sentenceType);
+ AnnotationSet sentences = inputAS.get(SENTENCE_ANNOTATION_TYPE);
int nbrDone = 0;
int nbrSentences = sentences.size();
- for (Annotation sentence : sentences) {
- AnnotationSet tokenSet =
Utils.getContainedAnnotations(inputAS, sentence, tokenType);
- Sentence tokens = new Sentence(tokenSet, stringFeature, posFeature);
+ for (Annotation sentence : sentences) {
+ AnnotationSet tokenSet = Utils.getContainedAnnotations(inputAS,
sentence, TOKEN_ANNOTATION_TYPE);
+ Sentence tokens = new Sentence(tokenSet, TOKEN_STRING_FEATURE_NAME,
TOKEN_CATEGORY_FEATURE_NAME);
String[] strings = tokens.getStrings();
String[] posTags = tokens.getTags();
String[] chunkTags = chunker.chunk(strings, posTags);
-
-
-
+
+
+
for (int i=0 ; i < chunkTags.length ; i++) {
-
+
if (sameAS) {
// add feature to existing annotation
tokens.get(i).getFeatures().put(chunkFeature, chunkTags[i]);
@@ -92,7 +87,7 @@
fm.putAll(oldToken.getFeatures());
fm.put(chunkFeature, chunkTags[i]);
try {
- outputAS.add(start, end, tokenType, fm);
+ outputAS.add(start, end, TOKEN_ANNOTATION_TYPE, fm);
}
catch (InvalidOffsetException e) {
throw new ExecutionException(e);
@@ -103,22 +98,22 @@
this.getName() + " has been abruptly interrupted!");
}
} // end loop over chunk tags within one sentence
-
+
nbrDone++;
fireProgressChanged((int)(100 * nbrDone / nbrSentences));
- } // end for sentence : sentences
-
+ } // end for sentence : sentences
+
fireProcessFinished();
fireStatusChanged("Finished " + this.getName() + " on " +
document.getName()
+ " in " + NumberFormat.getInstance().format(
(double)(System.currentTimeMillis() - startTime) / 1000)
- + " seconds!");
- }
-
-
-
- @Override
- public Resource init() throws ResourceInstantiationException {
+ + " seconds!");
+ }
+
+
+
+ @Override
+ public Resource init() throws ResourceInstantiationException {
InputStream modelInput = null;
try {
modelInput = modelUrl.openStream();
@@ -139,22 +134,22 @@
}
}
}
-
+
super.init();
return this;
- }
+ }
-
-
- /* CREOLE PARAMETERS */
-
+
+
+ /* CREOLE PARAMETERS */
+
@RunTime
@CreoleParameter(defaultValue = "",
comment = "annotation set containing tokens and sentences")
public void setInputASName(String name) {
this.inputASName = name;
}
-
+
public String getInputASName() {
return this.inputASName;
}
@@ -165,24 +160,24 @@
public void setOutputASName(String name) {
this.outputASName = name;
}
-
+
public String getOutputASName() {
return this.outputASName;
}
-
+
@RunTime
@CreoleParameter(defaultValue = "chunk",
comment = "feature for chunk tags")
public void setChunkFeature(String name) {
this.chunkFeature = name;
}
-
+
public String getChunkFeature() {
return this.chunkFeature;
}
-
+
@CreoleParameter(defaultValue = "models/english/en-chunker.bin",
comment = "location of the tagger model")
public void setModel(URL model) {
Modified: gate/trunk/plugins/OpenNLP/src/gate/opennlp/OpenNlpPOS.java
===================================================================
--- gate/trunk/plugins/OpenNLP/src/gate/opennlp/OpenNlpPOS.java 2012-11-14
02:22:00 UTC (rev 16274)
+++ gate/trunk/plugins/OpenNLP/src/gate/opennlp/OpenNlpPOS.java 2012-11-14
12:52:58 UTC (rev 16275)
@@ -39,121 +39,115 @@
private static final long serialVersionUID = 4010938787910114221L;
private static final Logger logger = Logger.getLogger(OpenNlpPOS.class);
-
- /* CREOLE PARAMETERS & SUCH*/
+ /* CREOLE PARAMETERS & WRAPPED COMPONENTS */
private String inputASName, outputASName;
private URL modelUrl;
private POSModel model;
private POSTaggerME tagger;
- private String tokenType = ANNIEConstants.TOKEN_ANNOTATION_TYPE;
- private String sentenceType = ANNIEConstants.SENTENCE_ANNOTATION_TYPE;
- private String stringFeature = ANNIEConstants.TOKEN_STRING_FEATURE_NAME;
- private String posFeature = ANNIEConstants.TOKEN_CATEGORY_FEATURE_NAME;
@Override
- public void execute() throws ExecutionException {
+ public void execute() throws ExecutionException {
interrupted = false;
long startTime = System.currentTimeMillis();
- if(document == null) {
- throw new ExecutionException("No document to process!");
- }
+ if(document == null) { throw new ExecutionException(
+ "No document to process!"); }
fireStatusChanged("Running " + this.getName() + " on " +
document.getName());
fireProgressChanged(0);
-
- AnnotationSet inputAS = document.getAnnotations(inputASName);
- AnnotationSet outputAS = document.getAnnotations(outputASName);
- boolean sameAS = inputAS.equals(outputAS);
- AnnotationSet sentences = inputAS.get(sentenceType);
- int nbrDone = 0;
- int nbrSentences = sentences.size();
+ AnnotationSet inputAS = document.getAnnotations(inputASName);
+ AnnotationSet outputAS = document.getAnnotations(outputASName);
+ boolean sameAS = inputAS.equals(outputAS);
- for (Annotation sentence : sentences) {
- AnnotationSet tokenSet =
Utils.getContainedAnnotations(inputAS, sentence, tokenType);
- Sentence tokens = new Sentence(tokenSet, stringFeature, null);
- String[] strings = tokens.getStrings();
+ AnnotationSet sentences = inputAS.get(SENTENCE_ANNOTATION_TYPE);
+ int nbrDone = 0;
+ int nbrSentences = sentences.size();
- if (strings.length > 0) {
- /* Run the OpenNLP tagger on this sentence,
- * then apply the tags. */
- String[] tags = tagger.tag(strings);
-
- for (int i=0 ; i < tags.length ; i++) {
- if (sameAS) {
- // add feature to existing annotation
- tokens.get(i).getFeatures().put(posFeature, tags[i]);
- }
- else {
- // new annotation with old features and new one
- Annotation oldToken = tokens.get(i);
- long start = oldToken.getStartNode().getOffset();
- long end = oldToken.getEndNode().getOffset();
- FeatureMap fm = Factory.newFeatureMap();
- fm.putAll(oldToken.getFeatures());
- fm.put(posFeature, tags[i]);
- try {
- outputAS.add(start, end, tokenType, fm);
- }
- catch (InvalidOffsetException e) {
- throw new ExecutionException(e);
- }
- }
- } // for loop applying tags
- } // if strings is not empty
-
- if(isInterrupted()) {
- throw new ExecutionInterruptedException("Execution of " +
- this.getName() + " has been abruptly interrupted!");
- }
+ for(Annotation sentence : sentences) {
+ AnnotationSet tokenSet =
+ Utils.getContainedAnnotations(inputAS, sentence,
+ TOKEN_ANNOTATION_TYPE);
+ Sentence tokens = new Sentence(tokenSet, TOKEN_STRING_FEATURE_NAME,
null);
+ String[] strings = tokens.getStrings();
+
+ if(strings.length > 0) {
+ /*
+ * Run the OpenNLP tagger on this sentence, then apply the tags.
+ */
+ String[] tags = tagger.tag(strings);
+
+ for(int i = 0; i < tags.length; i++) {
+ if(sameAS) {
+ // add feature to existing annotation
+ tokens.get(i).getFeatures()
+ .put(TOKEN_CATEGORY_FEATURE_NAME, tags[i]);
+ } else {
+ // new annotation with old features and new one
+ Annotation oldToken = tokens.get(i);
+ long start = oldToken.getStartNode().getOffset();
+ long end = oldToken.getEndNode().getOffset();
+ FeatureMap fm = Factory.newFeatureMap();
+ fm.putAll(oldToken.getFeatures());
+ fm.put(TOKEN_CATEGORY_FEATURE_NAME, tags[i]);
+ try {
+ outputAS.add(start, end, TOKEN_ANNOTATION_TYPE, fm);
+ } catch(InvalidOffsetException e) {
+ throw new ExecutionException(e);
+ }
+ }
+ } // for loop applying tags
+ } // if strings is not empty
+
+ if(isInterrupted()) { throw new ExecutionInterruptedException(
+ "Execution of " + this.getName() + " has been abruptly
interrupted!"); }
nbrDone++;
fireProgressChanged((int)(100 * nbrDone / nbrSentences));
- } // for sentence : sentences
-
+ } // for sentence : sentences
+
fireProcessFinished();
- fireStatusChanged("Finished " + this.getName() + " on " +
document.getName()
- + " in " + NumberFormat.getInstance().format(
+ fireStatusChanged("Finished "
+ + this.getName()
+ + " on "
+ + document.getName()
+ + " in "
+ + NumberFormat.getInstance().format(
(double)(System.currentTimeMillis() - startTime) / 1000)
+ " seconds!");
- }
+ }
- @Override
- public Resource init() throws ResourceInstantiationException {
+ @Override
+ public Resource init() throws ResourceInstantiationException {
InputStream modelInput = null;
try {
modelInput = modelUrl.openStream();
this.model = new POSModel(modelInput);
this.tagger = new POSTaggerME(model);
logger.info("OpenNLP POS Tagger: " + modelUrl.toString());
- }
- catch(IOException e) {
+ } catch(IOException e) {
throw new ResourceInstantiationException(e);
- }
- finally {
- if (modelInput != null) {
+ } finally {
+ if(modelInput != null) {
try {
modelInput.close();
- }
- catch (IOException e) {
+ } catch(IOException e) {
throw new ResourceInstantiationException(e);
}
}
}
-
+
super.init();
return this;
- }
+ }
+
+ @Override
+ public void reInit() throws ResourceInstantiationException {
+ init();
+ }
- @Override
- public void reInit() throws ResourceInstantiationException {
- init();
- }
-
-
/* CREOLE PARAMETERS */
@RunTime
Modified: gate/trunk/plugins/OpenNLP/src/gate/opennlp/OpenNlpSentenceSplit.java
===================================================================
--- gate/trunk/plugins/OpenNLP/src/gate/opennlp/OpenNlpSentenceSplit.java
2012-11-14 02:22:00 UTC (rev 16274)
+++ gate/trunk/plugins/OpenNLP/src/gate/opennlp/OpenNlpSentenceSplit.java
2012-11-14 12:52:58 UTC (rev 16275)
@@ -20,7 +20,8 @@
import java.io.*;
import java.net.URL;
import java.text.NumberFormat;
-
+import java.util.Arrays;
+import java.util.regex.*;
import opennlp.tools.sentdetect.*;
import opennlp.tools.util.Span;
import org.apache.log4j.Logger;
@@ -39,99 +40,128 @@
private static final Logger logger =
Logger.getLogger(OpenNlpSentenceSplit.class);
- /* CREOLE PARAMETERS & SUCH*/
+ /* CREOLE PARAMETERS & WRAPPED COMPONENTS */
private String annotationSetName = null;
+ private URL modelUrl;
private SentenceDetectorME splitter = null;
private SentenceModel model = null;
- private URL modelUrl;
- private String sentenceType = ANNIEConstants.SENTENCE_ANNOTATION_TYPE;
+
+ private Pattern punctEnding;
+
-
- @Override
- public void execute() throws ExecutionException {
+ @Override
+ public void execute() throws ExecutionException {
interrupted = false;
long startTime = System.currentTimeMillis();
- if(document == null) {
- throw new ExecutionException("No document to process!");
- }
+ if(document == null) { throw new ExecutionException(
+ "No document to process!"); }
fireStatusChanged("Running " + this.getName() + " on " +
document.getName());
fireProgressChanged(0);
-
- AnnotationSet annotations =
document.getAnnotations(annotationSetName);
- String text = document.getContent().toString();
- checkInterruption();
- Span[] spans = splitter.sentPosDetect(text);
- for (Span span : spans) {
- checkInterruption();
- FeatureMap fm = Factory.newFeatureMap();
- fm.put("source", "OpenNLP");
- long start = (long) span.getStart();
- long end = (long) span.getEnd();
+ AnnotationSet annotations = document.getAnnotations(annotationSetName);
+ String text = document.getContent().toString();
+ checkInterruption();
+ Span[] spans = splitter.sentPosDetect(text);
+ Arrays.sort(spans);
- try {
- annotations.add(start, end, sentenceType, fm);
- }
- catch (InvalidOffsetException e) {
- throw new ExecutionException(e);
- }
- }
-
- //TODO: generate Split annotations.
-
- //TODO: maybe generate SpaceToken annotations where the
- // Sentence ones don't meet.
-
+ for(int i = 0; i < spans.length; i++) {
+ Span span = spans[i];
+ checkInterruption();
+ FeatureMap fm = Factory.newFeatureMap();
+ fm.put("source", "OpenNLP");
+ FeatureMap sfm = Factory.newFeatureMap();
+ sfm.put("source", "OpenNLP");
+ long start = (long)span.getStart();
+ long end = (long)span.getEnd();
+ long splitStart, splitEnd;
+ String splitKind;
+
+ /*
+ * If the Sentence ends with 1+ punctuation marks, cover them with an
+ * internal Split. Otherwise, put an external Split between this Sentence
+ * and the next one (or the document end, if this is the last Sentence
---
+ * this can produce a 0-width Split, sorry).
+ */
+ String sentenceContent = Utils.stringFor(document, start, end);
+ Matcher matcher = punctEnding.matcher(sentenceContent);
+ if(matcher.find()) {
+ splitStart = start + (long)matcher.start();
+ splitEnd = end;
+ splitKind = "internal";
+ } else {
+ splitStart = end;
+ splitEnd = nextStart(spans, i);
+ splitKind = "external";
+ }
+
+ try {
+ annotations.add(start, end, SENTENCE_ANNOTATION_TYPE, fm);
+ sfm.put("kind", splitKind);
+ annotations.add(splitStart, splitEnd, "Split", sfm);
+ } catch(InvalidOffsetException e) {
+ throw new ExecutionException(e);
+ }
+ }
+
fireProcessFinished();
- fireStatusChanged("Finished " + this.getName() + " on " +
document.getName()
- + " in " + NumberFormat.getInstance().format(
+ fireStatusChanged("Finished "
+ + this.getName()
+ + " on "
+ + document.getName()
+ + " in "
+ + NumberFormat.getInstance().format(
(double)(System.currentTimeMillis() - startTime) / 1000)
+ " seconds!");
- }
+ }
+ private long nextStart(Span[] spans, int currentSpan) {
+ if(currentSpan < (spans.length - 1)) { return spans[currentSpan + 1]
+ .getStart(); }
+ // implied else: we're working on the last sentence
+ return this.document.getContent().size();
+ }
- @Override
- public Resource init() throws ResourceInstantiationException {
- InputStream modelInput = null;
- try {
- modelInput = modelUrl.openStream();
- this.model = new SentenceModel(modelInput);
- this.splitter = new SentenceDetectorME(model);
- logger.info("OpenNLP Splitter: " + modelUrl.toString());
- }
- catch(IOException e) {
- throw new ResourceInstantiationException(e);
- }
- finally {
- if (modelInput != null) {
- try {
- modelInput.close();
- }
- catch (IOException e) {
- throw new ResourceInstantiationException(e);
- }
- }
- }
-
- super.init();
- return this;
- }
+ @Override
+ public Resource init() throws ResourceInstantiationException {
+ /* TODO: This isn't perfect; it matches all punctuation at the
+ * end of a Sentence annotation, e.g., ").". */
+ punctEnding = Pattern.compile("\\p{Punct}+$");
-
- @Override
- public void reInit() throws ResourceInstantiationException {
- init();
- }
+ InputStream modelInput = null;
+ try {
+ modelInput = modelUrl.openStream();
+ this.model = new SentenceModel(modelInput);
+ this.splitter = new SentenceDetectorME(model);
+ logger.info("OpenNLP Splitter: " + modelUrl.toString());
+ } catch(IOException e) {
+ throw new ResourceInstantiationException(e);
+ } finally {
+ if(modelInput != null) {
+ try {
+ modelInput.close();
+ } catch(IOException e) {
+ throw new ResourceInstantiationException(e);
+ }
+ }
+ }
-
- private void checkInterruption() throws ExecutionInterruptedException {
- if(isInterrupted()) {
- throw new ExecutionInterruptedException("Execution of " +
- this.getName() + " has been abruptly interrupted!");
- }
- }
+ super.init();
+ return this;
+ }
+
+ @Override
+ public void reInit() throws ResourceInstantiationException {
+ init();
+ }
+
+
+ private void checkInterruption() throws ExecutionInterruptedException {
+ if(isInterrupted()) { throw new ExecutionInterruptedException(
+ "Execution of " + this.getName() + " has been abruptly interrupted!");
}
+ }
+
/* CREOLE PARAMETERS */
@RunTime
Modified: gate/trunk/plugins/OpenNLP/src/gate/opennlp/OpenNlpTokenizer.java
===================================================================
--- gate/trunk/plugins/OpenNLP/src/gate/opennlp/OpenNlpTokenizer.java
2012-11-14 02:22:00 UTC (rev 16274)
+++ gate/trunk/plugins/OpenNLP/src/gate/opennlp/OpenNlpTokenizer.java
2012-11-14 12:52:58 UTC (rev 16275)
@@ -38,116 +38,114 @@
private static final long serialVersionUID = 6965074842061250720L;
private static final Logger logger =
Logger.getLogger(OpenNlpTokenizer.class);
-
- /* CREOLE PARAMETERS & SUCH*/
- private String annotationSetName = null;
- private TokenizerME tokenizer = null;
- private TokenizerModel model = null;
- private URL modelUrl;
- private String tokenType = ANNIEConstants.TOKEN_ANNOTATION_TYPE;
- private String spaceTokenType = ANNIEConstants.SPACE_TOKEN_ANNOTATION_TYPE;
- private String stringFeature = ANNIEConstants.TOKEN_STRING_FEATURE_NAME;
-
-
+ /* CREOLE PARAMETERS & WRAPPED COMPONENTS */
+ private String annotationSetName = null;
+ private URL modelUrl;
+ private TokenizerME tokenizer = null;
+ private TokenizerModel model = null;
- public void execute() throws ExecutionException {
+
+ public void execute() throws ExecutionException {
interrupted = false;
long startTime = System.currentTimeMillis();
- if(document == null) {
- throw new ExecutionException("No document to process!");
- }
+ if(document == null) { throw new ExecutionException(
+ "No document to process!"); }
fireStatusChanged("Running " + this.getName() + " on " +
document.getName());
fireProgressChanged(0);
AnnotationSet annotations = document.getAnnotations(annotationSetName);
- String text = document.getContent().toString();
+ String text = document.getContent().toString();
checkInterruption();
- Span[] spans = tokenizer.tokenizePos(text);
-
- /* The spans ought to be ordered, but the OpenNLP
- * API is unclear. We need to be sure they are in
- * order so we can spot the gaps and put Space
- * Token annotations on them. */
- Arrays.sort(spans);
- long previousEnd = 0;
-
- for (Span span : spans) {
- checkInterruption();
- long start = (long) span.getStart();
- long end = (long) span.getEnd();
+ Span[] spans = tokenizer.tokenizePos(text);
- if (start > previousEnd) {
+ /*
+ * The spans ought to be ordered, but the OpenNLP API is unclear. We need
to
+ * be sure they are in order so we can spot the gaps and put Space Token
+ * annotations on them.
+ */
+ Arrays.sort(spans);
+ int previousEnd = 0;
+
+ for(Span span : spans) {
+ checkInterruption();
+ int tokenStart = span.getStart();
+ int tokenEnd = span.getEnd();
+
+ if(tokenStart > previousEnd) {
FeatureMap sfm = Factory.newFeatureMap();
sfm.put("source", "OpenNLP");
+ sfm.put(TOKEN_STRING_FEATURE_NAME,
+ text.substring(previousEnd, tokenStart));
+ sfm.put(TOKEN_LENGTH_FEATURE_NAME, tokenStart - previousEnd);
try {
- annotations.add(previousEnd, start, spaceTokenType, sfm);
- }
- catch (InvalidOffsetException e) {
+ annotations.add((long)previousEnd, (long)tokenStart,
+ SPACE_TOKEN_ANNOTATION_TYPE, sfm);
+ } catch(InvalidOffsetException e) {
throw new ExecutionException(e);
}
}
-
- previousEnd = end;
-
- FeatureMap fm = Factory.newFeatureMap();
- fm.put("source", "OpenNLP");
- fm.put(stringFeature, text.substring(span.getStart(),
span.getEnd()));
- try {
- annotations.add(start, end, tokenType, fm);
- }
- catch (InvalidOffsetException e) {
- throw new ExecutionException(e);
- }
- }
- fireProcessFinished();
- fireStatusChanged("Finished " + this.getName() + " on " +
document.getName()
- + " in " + NumberFormat.getInstance().format(
+ previousEnd = tokenEnd;
+
+ FeatureMap fm = Factory.newFeatureMap();
+ fm.put("source", "OpenNLP");
+ fm.put(TOKEN_STRING_FEATURE_NAME, text.substring(tokenStart, tokenEnd));
+ fm.put(TOKEN_LENGTH_FEATURE_NAME, span.length());
+ try {
+ annotations.add((long)tokenStart, (long)tokenEnd,
+ TOKEN_ANNOTATION_TYPE, fm);
+ } catch(InvalidOffsetException e) {
+ throw new ExecutionException(e);
+ }
+ }
+
+ fireProcessFinished();
+ fireStatusChanged("Finished "
+ + this.getName()
+ + " on "
+ + document.getName()
+ + " in "
+ + NumberFormat.getInstance().format(
(double)(System.currentTimeMillis() - startTime) / 1000)
+ " seconds!");
- }
+ }
+
+ private void checkInterruption() throws ExecutionInterruptedException {
+ if(isInterrupted()) { throw new ExecutionInterruptedException(
+ "Execution of " + this.getName() + " has been abruptly interrupted!");
}
+ }
- private void checkInterruption() throws ExecutionInterruptedException {
- if(isInterrupted()) {
- throw new ExecutionInterruptedException("Execution of " +
- this.getName() + " has been abruptly interrupted!");
- }
- }
-
-
- public Resource init() throws ResourceInstantiationException {
- InputStream modelInput = null;
+
+ public Resource init() throws ResourceInstantiationException {
+ InputStream modelInput = null;
try {
modelInput = modelUrl.openStream();
this.model = new TokenizerModel(modelInput);
this.tokenizer = new TokenizerME(model);
logger.info("OpenNLP Tokenizer: " + modelUrl.toString());
- }
- catch(IOException e) {
+ } catch(IOException e) {
throw new ResourceInstantiationException(e);
- }
- finally {
- if (modelInput != null) {
+ } finally {
+ if(modelInput != null) {
try {
modelInput.close();
- }
- catch (IOException e) {
+ } catch(IOException e) {
throw new ResourceInstantiationException(e);
}
}
}
-
- super.init();
- return this;
- }
+ super.init();
+ return this;
+ }
+
- public void reInit() throws ResourceInstantiationException {
- init();
- }
+ public void reInit() throws ResourceInstantiationException {
+ init();
+ }
-
+
/* CREOLE PARAMETERS */
@RunTime
Modified: gate/trunk/plugins/OpenNLP/src/gate/opennlp/Sentence.java
===================================================================
--- gate/trunk/plugins/OpenNLP/src/gate/opennlp/Sentence.java 2012-11-14
02:22:00 UTC (rev 16274)
+++ gate/trunk/plugins/OpenNLP/src/gate/opennlp/Sentence.java 2012-11-14
12:52:58 UTC (rev 16275)
@@ -67,7 +67,7 @@
* @param i
* @return the GATE-style start offset in characters
*/
-public long getStartOffset(Span span) {
+ public long getStartOffset(Span span) {
int start = span.getStart();
if ( (start >= 0) && (start < size) ) {
return this.tokens.get(start).getStartNode().getOffset();
This was sent by the SourceForge.net collaborative development platform, the
world's largest Open Source development site.
------------------------------------------------------------------------------
Monitor your physical, virtual and cloud infrastructure from a single
web console. Get in-depth insight into apps, servers, databases, vmware,
SAP, cloud infrastructure, etc. Download 30-day Free Trial.
Pricing starts from $795 for 25 servers or applications!
http://p.sf.net/sfu/zoho_dev2dev_nov
_______________________________________________
GATE-cvs mailing list
[email protected]
https://lists.sourceforge.net/lists/listinfo/gate-cvs
