Revision: 18928
http://sourceforge.net/p/gate/code/18928
Author: ian_roberts
Date: 2015-09-30 17:32:02 +0000 (Wed, 30 Sep 2015)
Log Message:
-----------
Bounds checking to avoid adding a zero-length Token
Modified Paths:
--------------
gate/trunk/plugins/Lang_French/tokeniser/postprocess.jape
Modified: gate/trunk/plugins/Lang_French/tokeniser/postprocess.jape
===================================================================
--- gate/trunk/plugins/Lang_French/tokeniser/postprocess.jape 2015-09-30
01:19:56 UTC (rev 18927)
+++ gate/trunk/plugins/Lang_French/tokeniser/postprocess.jape 2015-09-30
17:32:02 UTC (rev 18928)
@@ -78,11 +78,14 @@
/* offset += dashIndex; */ // <-- REMOVE THIS LINE
startIndex = dashIndex + 1;
}
- features = Factory.newFeatureMap();
- features.putAll(annotation.getFeatures());
- features.put("string", content.substring(startIndex));
- features.put("length", content.length()-startIndex);
- outputAS.add(offset+startIndex, endOffset, "Token", features); // <-- MODIF
HERE
+ if(content.length() < startIndex) {
+ // there is trailing content, make an extra token
+ features = Factory.newFeatureMap();
+ features.putAll(annotation.getFeatures());
+ features.put("string", content.substring(startIndex));
+ features.put("length", content.length()-startIndex);
+ outputAS.add(offset+startIndex, endOffset, "Token", features); // <--
MODIF HERE
+ }
} catch (InvalidOffsetException e) {
throw new LuckyException(e);
}
This was sent by the SourceForge.net collaborative development platform, the
world's largest Open Source development site.
------------------------------------------------------------------------------
_______________________________________________
GATE-cvs mailing list
GATE-cvs@lists.sourceforge.net
https://lists.sourceforge.net/lists/listinfo/gate-cvs
