Revision: 14568
http://gate.svn.sourceforge.net/gate/?rev=14568&view=rev
Author: adamfunk
Date: 2011-11-17 20:15:26 +0000 (Thu, 17 Nov 2011)
Log Message:
-----------
Modified Flexible Gazetteer to map Lookups on the temp document
loosely to the Tokens (typically) on the real document, so that
features' values with non-word characters don't cause the
InvalidOffsetException in the transfer back to the original document.
Made everything type-safe too.
Modified Paths:
--------------
gate/trunk/src/gate/creole/gazetteer/FlexibleGazetteer.java
gate/trunk/src/gate/creole/gazetteer/NodePosition.java
Modified: gate/trunk/src/gate/creole/gazetteer/FlexibleGazetteer.java
===================================================================
--- gate/trunk/src/gate/creole/gazetteer/FlexibleGazetteer.java 2011-11-17
16:27:55 UTC (rev 14567)
+++ gate/trunk/src/gate/creole/gazetteer/FlexibleGazetteer.java 2011-11-17
20:15:26 UTC (rev 14568)
@@ -1,7 +1,7 @@
-/*
+0;271;0c/*
* FlexibleGazetteer.java
*
- * Copyright (c) 2004, The University of Sheffield.
+ * Copyright (c) 2004-2011, The University of Sheffield.
*
* This file is part of GATE (see http://gate.ac.uk/), and is free
* software, licenced under the GNU Library General Public License,
@@ -11,6 +11,7 @@
* licence.html, and is also available at http://gate.ac.uk/gate/licence.html.
*
* Niraj Aswani 02/2002
+ * $Id$
*
*/
@@ -66,15 +67,10 @@
* @version 1.0
*/
-public class FlexibleGazetteer extends AbstractLanguageAnalyser implements
-
ProcessingResource {
+public class FlexibleGazetteer extends AbstractLanguageAnalyser
+implements ProcessingResource {
- /**
- * Constructor
- */
- public FlexibleGazetteer() {
- changedNodes = new ArrayList();
- }
+ private static final long serialVersionUID = -1023682327651886920L;
/**
* Does the actual loading and parsing of the lists. This method must
@@ -85,6 +81,7 @@
if(gazetteerInst == null)
throw new ResourceInstantiationException("No Gazetteer Provided!");
+ this.changedNodes = new ArrayList<NodePosition>();
return this;
}
@@ -93,7 +90,7 @@
* parameters are set. If they are not, an exception will be fired.
*/
public void execute() throws ExecutionException {
- changedNodes = new ArrayList();
+ changedNodes = new ArrayList<NodePosition>();
fireProgressChanged(0);
fireStatusChanged("Checking Document...");
if(document == null) {
@@ -101,30 +98,20 @@
}
fireStatusChanged("Creating temporary Document...");
- StringBuffer newdocString = new StringBuffer(document.getContent()
- .toString());
+ StringBuffer newdocString = new
StringBuffer(document.getContent().toString());
Document tempDoc = null;
- boolean chineseSplit = false;
if(inputFeatureNames == null || inputFeatureNames.size() == 0) {
- inputFeatureNames = new ArrayList();
+ inputFeatureNames = new ArrayList<String>();
}
- Iterator tokenIter = getSortedAnnotationIterator(document,
inputAnnotationSetName);
long totalDeductedSpaces = 0;
fireStatusChanged("Replacing contents with the feature value...");
+ outer: for (Annotation currentToken :
Utils.inDocumentOrder(document.getAnnotations(inputAnnotationSetName))) {
+ // check if it is a chinesesplit; if it is, replace no space character
with a single space
+
if(currentToken.getType().equals(ANNIEConstants.SPACE_TOKEN_ANNOTATION_TYPE)
+ &&
((String)(currentToken.getFeatures().get(ANNIEConstants.TOKEN_KIND_FEATURE_NAME))).equals("ChineseSplit"))
{
- outer: while(tokenIter != null && tokenIter.hasNext()) {
- Annotation currentToken = (Annotation)tokenIter.next();
-
- // check if it is a chinesesplit
- // if it is, replace no space character with a single space
- if(currentToken.getType().equals(
- ANNIEConstants.SPACE_TOKEN_ANNOTATION_TYPE)
- && ((String)(currentToken.getFeatures()
- .get(ANNIEConstants.TOKEN_KIND_FEATURE_NAME)))
- .equals("ChineseSplit")) {
-
// for chinese split startnode and end node are same
long startOffset = currentToken.getStartNode().getOffset().longValue();
@@ -134,7 +121,6 @@
long newEndOffset = newStartOffset + 1;
NodePosition newNode = new NodePosition(startOffset, startOffset,
newStartOffset, newEndOffset, totalDeductedSpaces);
- chineseSplit = true;
// here is the addition of space in the document
totalDeductedSpaces--;
@@ -146,58 +132,48 @@
// search in the provided inputFeaturesNames
// if the current annotation has a feature value that user
// wants to paste on and replace the original string
- inner: for(int i = 0; i < inputFeatureNames.size(); i++) {
- String[] keyVal = ((String)(inputFeatureNames.get(i))).split("[.]");
+ inner: for(String inputFeatureName : inputFeatureNames) {
+ String[] keyVal = inputFeatureName .split("[.]");
if(keyVal.length == 2) {
- // val is the feature name
- // key is the annotationName
- if(currentToken.getType().equals(keyVal[0])) {
- FeatureMap features = currentToken.getFeatures();
- String newTokenValue = (String)(features.get(keyVal[1]));
-
- // what if provided feature doesnot exist
- if(newTokenValue == null) {
- continue;
-
- }
- else {
- // feature value found so we need to replace it
- // find the start and end offsets for this token
- long startOffset = currentToken.getStartNode().getOffset()
- .longValue();
- long endOffset = currentToken.getEndNode().getOffset()
- .longValue();
-
- // replacement code start
- long actualLength = endOffset - startOffset;
- // let us find the difference between the lengths of the
- // actual string and the newTokenValue
- long lengthDifference = actualLength - newTokenValue.length();
-
- // replacement code end
-
- // so lets find out the new startOffset and endOffset
- long newStartOffset = startOffset - totalDeductedSpaces;
- long newEndOffset = newStartOffset + newTokenValue.length();
- totalDeductedSpaces += lengthDifference;
-
- // and make the entry for this
- NodePosition newNode = new NodePosition(startOffset, endOffset,
- newStartOffset, newEndOffset, totalDeductedSpaces);
- changedNodes.add(newNode);
-
- // and finally replace the actual string in the document
- // with the new document
- newdocString = newdocString.replace((int)newStartOffset,
- (int)newStartOffset + (int)actualLength, // replacement
code
- newTokenValue);
- break inner;
- }
+ // keyVal[0] = annotation type
+ // keyVal[1] = feature name
+ if(currentToken.getType().equals(keyVal[0]) &&
currentToken.getFeatures().containsKey(keyVal[1])) {
+ String newTokenValue =
currentToken.getFeatures().get(keyVal[1]).toString();
+
+ // feature value found so we need to replace it
+ // find the start and end offsets for this token
+ long startOffset =
currentToken.getStartNode().getOffset().longValue();
+ long endOffset = currentToken.getEndNode().getOffset().longValue();
+
+ // replacement code start
+ long actualLength = endOffset - startOffset;
+ // let us find the difference between the lengths of the
+ // actual string and the newTokenValue
+ long lengthDifference = actualLength - newTokenValue.length();
+
+ // replacement code end
+
+ // so lets find out the new startOffset and endOffset
+ long newStartOffset = startOffset - totalDeductedSpaces;
+ long newEndOffset = newStartOffset + newTokenValue.length();
+ totalDeductedSpaces += lengthDifference;
+
+ // and make the entry for this
+ NodePosition newNode = new NodePosition(startOffset, endOffset,
+ newStartOffset, newEndOffset, totalDeductedSpaces);
+ changedNodes.add(newNode);
+
+ // and finally replace the actual string in the document
+ // with the new document
+ newdocString = newdocString.replace((int)newStartOffset,
+ (int)newStartOffset + (int)actualLength, // replacement code
+ newTokenValue);
+ break inner;
}
}
- }
- }
+ } // END OF "inner" LOOP
+ } // END OF "outer" LOOP
fireStatusChanged("New Document to be processed with Gazetteer...");
try {
@@ -218,7 +194,7 @@
}
// lets create the gazetteer based on the provided gazetteer name
- FeatureMap params = Factory.newFeatureMap();
+ //FeatureMap params = Factory.newFeatureMap();
gazetteerInst.setDocument(tempDoc);
gazetteerInst.setAnnotationSetName(this.outputAnnotationSetName);
@@ -231,61 +207,48 @@
}
// now the tempDoc has been looked up, we need to shift the tokens
- // from
- // this temp document to the original document
+ // from this temp document to the original document
fireStatusChanged("Transfering new tags to the original one...");
- Iterator lookupIter = getSortedAnnotationIterator(tempDoc,
outputAnnotationSetName);
- AnnotationSet original = (outputAnnotationSetName == null) ? document
- .getAnnotations() : document
- .getAnnotations(outputAnnotationSetName);
+ AnnotationSet original = document.getAnnotations(outputAnnotationSetName);
- int positionOfI = 0;
- while(lookupIter != null && lookupIter.hasNext()) {
- Annotation currentLookup = (Annotation)(lookupIter.next());
+ for (Annotation currentLookup :
Utils.inDocumentOrder(tempDoc.getAnnotations(outputAnnotationSetName))) {
long startOffset = currentLookup.getStartNode().getOffset().longValue();
long endOffset = currentLookup.getEndNode().getOffset().longValue();
- // if there was any change node before the startOffset
+ long originalStart = 0;
+ long originalEnd = tempDoc.getContent().size() - 1L;
- NodePosition toUse = null;
- int i = positionOfI;
+ int i = 0;
for(; i < changedNodes.size(); i++) {
- NodePosition np = (NodePosition)changedNodes.get(i);
+ NodePosition np = changedNodes.get(i);
- // continue until we find a node whose new end node has a value
- // greater than or equal to the current lookup
- if(np.getNewStartNode() < startOffset) {
- positionOfI = i;
- toUse = np;
- continue;
- } else {
+ // Find the last node whose temp start node is less than or equal
+ // to the temp lookup's start
+ if(np.getNewStartNode() <= startOffset) {
+ originalStart = np.getOldStartNode();
+ }
+ else {
break;
}
}
- long spacesToAddToSO = toUse != null ? toUse.getDeductedSpaces() : 0;
+ i--;
- toUse = null;
for(; i < changedNodes.size(); i++) {
- NodePosition np = (NodePosition)changedNodes.get(i);
+ NodePosition np = changedNodes.get(i);
- // continue until we find a node whose new end node has a value
- // less tgreater than or equal to the current lookup
- if(np.getNewStartNode() <= endOffset) {
- toUse = np;
- continue;
- } else {
+ // Find the first node whose temp end node is greater than or equal
+ // to the temp lookup's end
+ if(np.getNewEndNode() >= endOffset) {
+ originalEnd = np.getOldEndNode();
break;
}
}
-
- long spacesToAddToEO = toUse != null ? toUse.getDeductedSpaces() :
spacesToAddToSO;
-
- try {
- original.add(new Long(startOffset + spacesToAddToSO), new Long(
- endOffset + spacesToAddToEO), currentLookup.getType(),
- currentLookup.getFeatures());
- }
+
+ try {
+ original.add(originalStart, originalEnd,
+ currentLookup.getType(), currentLookup.getFeatures());
+ } // This should no longer happen
catch(InvalidOffsetException ioe) {
throw new ExecutionException(ioe);
}
@@ -295,7 +258,7 @@
// now remove the newDoc
Factory.deleteResource(tempDoc);
fireProcessFinished();
- }
+ } // END execute METHOD
/**
* Sets the document to work on
@@ -359,7 +322,7 @@
*
* @param inputs
*/
- public void setInputFeatureNames(java.util.List inputs) {
+ public void setInputFeatureNames(java.util.List<String> inputs) {
this.inputFeatureNames = inputs;
}
@@ -369,7 +332,7 @@
*
* @return a {@link List} value.
*/
- public java.util.List getInputFeatureNames() {
+ public java.util.List<String> getInputFeatureNames() {
return this.inputFeatureNames;
}
@@ -381,32 +344,8 @@
this.gazetteerInst = gazetteerInst;
}
- /**
- * This method takes the document and the annotationSetName and then
- * creates a interator for the annotations available in the document
- * under the provided annotationSetName
- *
- * @param doc
- * @param annotationSetName
- * @return an {@link Iterator}
- */
- public Iterator getSortedAnnotationIterator(gate.Document doc, String
annotationSetName) {
- AnnotationSet inputAs = (annotationSetName == null)
- ? doc.getAnnotations()
- : doc.getAnnotations(annotationSetName);
- AnnotationSet tempSet = inputAs.get();
- if(tempSet == null) return null;
- List tokens = new ArrayList(inputAs.get());
- if(tokens == null) return null;
-
- Comparator offsetComparator = new OffsetComparator();
- Collections.sort(tokens, offsetComparator);
- Iterator tokenIter = tokens.listIterator();
- return tokenIter;
- }
-
// Gazetteer Runtime parameters
private gate.Document document;
@@ -417,8 +356,8 @@
// Flexible Gazetteer parameter
private Gazetteer gazetteerInst;
- private java.util.List inputFeatureNames;
+ private java.util.List<String> inputFeatureNames;
// parameters required within the program
- private ArrayList changedNodes;
+ private ArrayList<NodePosition> changedNodes;
}
Modified: gate/trunk/src/gate/creole/gazetteer/NodePosition.java
===================================================================
--- gate/trunk/src/gate/creole/gazetteer/NodePosition.java 2011-11-17
16:27:55 UTC (rev 14567)
+++ gate/trunk/src/gate/creole/gazetteer/NodePosition.java 2011-11-17
20:15:26 UTC (rev 14568)
@@ -19,7 +19,7 @@
/**
* <p>Title: NodePosition.java </p>
* <p>Description: This class is used to store the information about the
- * changes in the text and the addition or the substraction of the spaces.
+ * changes in the text and the addition or the subtraction of the spaces.
* It is used by FlexibleGazetteer. </p>
* @author Niraj Aswani
* @version 1.0
This was sent by the SourceForge.net collaborative development platform, the
world's largest Open Source development site.
------------------------------------------------------------------------------
All the data continuously generated in your IT infrastructure
contains a definitive record of customers, application performance,
security threats, fraudulent activity, and more. Splunk takes this
data and makes sense of it. IT sense. And common sense.
http://p.sf.net/sfu/splunk-novd2d
_______________________________________________
GATE-cvs mailing list
[email protected]
https://lists.sourceforge.net/lists/listinfo/gate-cvs
