This is an automated email from the ASF dual-hosted git repository.
mawiesne pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/opennlp.git
The following commit(s) were added to refs/heads/main by this push:
new d5d37dca2 OPENNLP-205: Refactor the end-of-sentence position to span
mapping in SentenceDetectorME (#1141)
d5d37dca2 is described below
commit d5d37dca2088aa41ec167c863ee368481b4a2bcc
Author: Kristian Rickert <[email protected]>
AuthorDate: Mon Jul 6 13:17:48 2026 -0400
OPENNLP-205: Refactor the end-of-sentence position to span mapping in
SentenceDetectorME (#1141)
---
.../tools/sentdetect/SentenceDetectorME.java | 161 ++++++-----
.../SentenceDetectorMESpanMappingTest.java | 308 +++++++++++++++++++++
2 files changed, 404 insertions(+), 65 deletions(-)
diff --git
a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/sentdetect/SentenceDetectorME.java
b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/sentdetect/SentenceDetectorME.java
index 074db9625..7be0d10b1 100644
---
a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/sentdetect/SentenceDetectorME.java
+++
b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/sentdetect/SentenceDetectorME.java
@@ -42,6 +42,7 @@ import opennlp.tools.util.Span;
import opennlp.tools.util.StringList;
import opennlp.tools.util.StringUtil;
import opennlp.tools.util.TrainingParameters;
+import opennlp.tools.util.normalizer.CharClass;
/**
* A sentence detector for splitting up raw text into sentences.
@@ -185,15 +186,36 @@ public class SentenceDetectorME implements
SentenceDetector, Probabilistic {
return sentences;
}
+ // The whitespace definition of the sentence detector: the Unicode
White_Space set
+ // (OPENNLP-205). It drives both the detection loop of sentPosDetect (the
delimiter-run skip
+ // heuristic and the placement of sentence-start positions) and the
position-to-span mapping,
+ // so both stages agree on what separates sentences. Unlike the previously
used
+ // StringUtil.isWhitespace, this covers the next line control (U+0085) and
does not treat the
+ // U+001C..U+001F information separators as whitespace; around those
characters the candidate
+ // positions themselves can differ from pre-OPENNLP-205 releases, not only
the span edges.
+ // Both deltas are pinned in SentenceDetectorMESpanMappingTest. Model
feature generation
+ // (SDContextGenerator) is not affected.
+ private static final CharClass WHITESPACE = CharClass.whitespace();
+
private int getFirstWS(CharSequence s, int pos) {
- while (pos < s.length() && !StringUtil.isWhitespace(s.charAt(pos)))
- pos++;
+ while (pos < s.length()) {
+ final int cp = Character.codePointAt(s, pos);
+ if (WHITESPACE.contains(cp)) {
+ break;
+ }
+ pos += Character.charCount(cp);
+ }
return pos;
}
private int getFirstNonWS(CharSequence s, int pos) {
- while (pos < s.length() && StringUtil.isWhitespace(s.charAt(pos)))
- pos++;
+ while (pos < s.length()) {
+ final int cp = Character.codePointAt(s, pos);
+ if (!WHITESPACE.contains(cp)) {
+ break;
+ }
+ pos += Character.charCount(cp);
+ }
return pos;
}
@@ -244,77 +266,86 @@ public class SentenceDetectorME implements
SentenceDetector, Probabilistic {
}
int[] starts = ArrayMath.toIntArray(positions);
+ Span[] spans = mapPositionsToSpans(s, starts, localProbs);
- // string does not contain sentence end positions
- if (starts.length == 0) {
-
- // remove leading and trailing whitespace
- int start = 0;
- int end = s.length();
-
- while (start < s.length() && StringUtil.isWhitespace(s.charAt(start)))
- start++;
-
- while (end > 0 && StringUtil.isWhitespace(s.charAt(end - 1)))
- end--;
-
- if (end - start > 0) {
- localProbs.add(1d);
- state.sentProbs = localProbs;
- return new Span[] {new Span(start, end)};
- }
- else {
- state.sentProbs = localProbs;
- return new Span[0];
- }
- }
-
- // Convert the sentence end indexes to spans
+ // Publish for backward-compatible probs() access (last-writer-wins under
concurrency)
+ state.sentProbs = localProbs;
- boolean leftover = starts[starts.length - 1] != s.length();
- Span[] spans = new Span[leftover ? starts.length + 1 : starts.length];
+ return spans;
+ }
+ /**
+ * Maps accepted sentence-start positions to trimmed sentence {@link Span}s,
the core of the
+ * end-of-sentence position to span mapping (OPENNLP-205). Not part of the
public API; the
+ * package visibility is deliberate, kept so the mapping can be exercised
directly in tests.
+ * Several branches (the whitespace-only candidate that keeps {@code probs}
aligned, and the
+ * stale-{@code probs} reset in the zero-positions branch) are not reachable
through the public
+ * {@code sentPosDetect} entry point.
+ *
+ * <p>Each span runs from the previous position (or the text start) to the
next position, with
+ * Unicode {@code White_Space} trimmed from both edges. A candidate that is
whitespace-only is
+ * dropped together with its probability, so {@code probs} and the returned
spans always stay
+ * aligned; text after the last position becomes a final span with
probability {@code 1.0}.
+ * With no positions at all, the whole text is one trimmed span with
probability {@code 1.0},
+ * or no span when it is blank.</p>
+ *
+ * @param s The text the positions refer to.
+ * @param starts The accepted sentence-start positions, ascending.
+ * @param probs The probability per position; mutated (in every branch) so
it exactly mirrors
+ * the probabilities of the returned spans.
+ * @return The trimmed spans, in order, each carrying its probability via
+ * {@link Span#getProb()}.
+ */
+ Span[] mapPositionsToSpans(CharSequence s, int[] starts, List<Double> probs)
{
+ final List<Span> spans = new ArrayList<>(starts.length + 1);
+ int sentStart = 0;
for (int si = 0; si < starts.length; si++) {
- int start;
-
- if (si == 0) {
- start = 0;
- }
- else {
- start = starts[si - 1];
- }
+ // A candidate might contain only whitespace; it is dropped together
with its probability,
+ // which keeps the spans and the probabilities aligned by construction.
+ addTrimmedSpan(spans, s, sentStart, starts[si], probs.get(si));
+ sentStart = starts[si];
+ }
+ // The text after the last position; with no positions at all this is the
whole text, which
+ // covers input without any accepted sentence end.
+ if (starts.length == 0 || starts[starts.length - 1] != s.length()) {
+ addTrimmedSpan(spans, s, sentStart, s.length(), 1d);
+ }
+ probs.clear();
+ for (Span span : spans) {
+ probs.add(span.getProb());
+ }
+ return spans.toArray(new Span[0]);
+ }
- // A span might contain only white spaces, in this case the length of
- // the span will be zero after trimming and should be ignored.
- Span span = new Span(start, starts[si]).trim(s);
- if (span.length() > 0) {
- spans[si] = span;
- }
- else {
- localProbs.remove(si);
- }
+ // Appends [start, end) as a span with the given probability attached,
unless it is
+ // whitespace-only and trims to nothing.
+ private void addTrimmedSpan(List<Span> spans, CharSequence s, int start, int
end,
+ double prob) {
+ Span span = trimmedSpan(s, start, end);
+ if (span != null) {
+ spans.add(new Span(span, prob));
}
+ }
- if (leftover) {
- Span span = new Span(starts[starts.length - 1], s.length()).trim(s);
- if (span.length() > 0) {
- spans[spans.length - 1] = span;
- localProbs.add(1d);
+ // Returns [start, end) with Unicode White_Space trimmed from both edges, or
null when nothing
+ // remains. Scans by code point, matching the CharClass discipline; all
current White_Space
+ // members are BMP, but this keeps surrogate pairs at the edges intact by
construction.
+ private Span trimmedSpan(CharSequence s, int start, int end) {
+ while (start < end) {
+ final int cp = Character.codePointAt(s, start);
+ if (!WHITESPACE.contains(cp)) {
+ break;
}
+ start += Character.charCount(cp);
}
- /*
- * set the prob for each span
- */
- for (int i = 0; i < spans.length; i++) {
- double prob = localProbs.get(i);
- spans[i] = new Span(spans[i], prob);
-
+ while (end > start) {
+ final int cp = Character.codePointBefore(s, end);
+ if (!WHITESPACE.contains(cp)) {
+ break;
+ }
+ end -= Character.charCount(cp);
}
-
- // Publish for backward-compatible probs() access (last-writer-wins under
concurrency)
- state.sentProbs = localProbs;
-
- return spans;
+ return end > start ? new Span(start, end) : null;
}
/**
diff --git
a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/sentdetect/SentenceDetectorMESpanMappingTest.java
b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/sentdetect/SentenceDetectorMESpanMappingTest.java
new file mode 100644
index 000000000..d081c1839
--- /dev/null
+++
b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/sentdetect/SentenceDetectorMESpanMappingTest.java
@@ -0,0 +1,308 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.tools.sentdetect;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Locale;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+
+import opennlp.tools.dictionary.Dictionary;
+import opennlp.tools.util.Span;
+
+/**
+ * Characterization tests for the end-of-sentence position to {@link Span}
mapping of
+ * {@link SentenceDetectorME#sentPosDetect(CharSequence)} (OPENNLP-205). These
pin the mapping
+ * behavior, including edge cases, so the mapping can be refactored without
silently changing
+ * results; every expectation was captured from the behavior of the
pre-refactoring implementation.
+ */
+public class SentenceDetectorMESpanMappingTest extends
AbstractSentenceDetectorTest {
+
+ private static final String SECOND = "There are many tests, this is the
second.";
+
+ private static SentenceDetectorME tokenEnd;
+ private static SentenceDetectorME noTokenEnd;
+
+ @BeforeAll
+ static void prepareResources() throws IOException {
+ Dictionary abb = loadAbbDictionary(Locale.ENGLISH);
+ tokenEnd = new SentenceDetectorME(
+ train(new SentenceDetectorFactory("eng", true, abb, null),
Locale.ENGLISH));
+ noTokenEnd = new SentenceDetectorME(
+ train(new SentenceDetectorFactory("eng", false, abb, null),
Locale.ENGLISH));
+ }
+
+ private static void assertSpans(SentenceDetectorME sd, String input, Span...
expected) {
+ Span[] actual = sd.sentPosDetect(input);
+ Assertions.assertEquals(expected.length, actual.length,
+ () -> "span count for: " + input);
+ for (int i = 0; i < expected.length; i++) {
+ Assertions.assertEquals(expected[i].getStart(), actual[i].getStart(),
"start of span " + i);
+ Assertions.assertEquals(expected[i].getEnd(), actual[i].getEnd(), "end
of span " + i);
+ }
+ // The probability list must stay aligned with the returned spans,
whatever the mapping drops.
+ Assertions.assertEquals(actual.length, sd.probs().length,
+ () -> "probs alignment for: " + input);
+ }
+
+ @Test
+ void twoSentencesMapToExactSpans() {
+ String input = "This is a test. " + SECOND;
+ assertSpans(tokenEnd, input, new Span(0, 15), new Span(16, 57));
+ assertSpans(noTokenEnd, input, new Span(0, 15), new Span(16, 57));
+ }
+
+ @Test
+ void leadingAndTrailingWhitespaceIsExcludedFromSpans() {
+ String input = " This is a test. " + SECOND + " ";
+ assertSpans(tokenEnd, input, new Span(3, 18), new Span(21, 62));
+ assertSpans(noTokenEnd, input, new Span(3, 18), new Span(21, 62));
+ }
+
+ @Test
+ void multiCharacterDelimiterRunStaysWithItsSentence() {
+ String input = "This is great!!! " + SECOND;
+ assertSpans(tokenEnd, input, new Span(0, 16), new Span(17, 58));
+ assertSpans(noTokenEnd, input, new Span(0, 16), new Span(17, 58));
+ }
+
+ @Test
+ void noBreakSpaceBetweenSentencesSplitsAndTrims() {
+ // NBSP is SPACE_SEPARATOR, recognized as whitespace by the mapping both
before and after the
+ // OPENNLP-205 refactoring; the second span must start at the word, not
the NBSP.
+ String input = "This is a test.\u00A0" + SECOND;
+ assertSpans(tokenEnd, input, new Span(0, 15), new Span(16, 57));
+ }
+
+ @Test
+ void carriageReturnLineFeedBetweenSentences() {
+ String input = "This is a test.\r\n" + SECOND;
+ assertSpans(tokenEnd, input, new Span(0, 15), new Span(17, 58));
+ }
+
+ @Test
+ void whitespaceOnlyAndEmptyInputYieldNoSpans() {
+ assertSpans(tokenEnd, " ");
+ assertSpans(tokenEnd, "");
+ }
+
+ @Test
+ void inputWithoutEndOfSentenceCharactersIsOneTrimmedSpan() {
+ assertSpans(tokenEnd, " no end of sentence marker here ", new Span(2,
32));
+ }
+
+ @Test
+ void delimiterIslandsRemainSeparateSpans() {
+ // Pinned, not endorsed: a free-standing delimiter between sentences is
reported as its own
+ // one-character span with its own probability. Changing this would change
public output, so
+ // the OPENNLP-205 mapping refactoring must preserve it.
+ String input = "This is a test. . " + SECOND;
+ assertSpans(tokenEnd, input, new Span(0, 15), new Span(16, 17), new
Span(18, 59));
+ assertSpans(noTokenEnd, input, new Span(0, 15), new Span(16, 17), new
Span(18, 59));
+ String runs = "This is a test. . . " + SECOND;
+ assertSpans(tokenEnd, runs,
+ new Span(0, 15), new Span(17, 18), new Span(20, 21), new Span(23, 64));
+ }
+
+ @Test
+ void nextLineControlBetweenSpacedSentencesIsTrimmed() {
+ // U+0085 NEL is Unicode White_Space. The pre-refactoring mapping missed it
+ // (StringUtil.isWhitespace does not cover it), so the second span used to
start at the NEL
+ // and carry it as leading content; with the mapping on the CharClass
Unicode White_Space set
+ // (OPENNLP-205), the span starts at the word.
+ String input = "This is a test. \u0085 " + SECOND;
+ assertSpans(tokenEnd, input, new Span(0, 15), new Span(18, 59));
+ }
+
+ @Test
+ void bareNextLineControlDoesNotSplit() {
+ // With no ordinary space around the delimiter, the model's context
(unchanged by
+ // OPENNLP-205; feature generation must stay stable for existing models)
does not produce a
+ // split here, so the text stays one span covering both parts.
+ String input = "This is a test.\u0085" + SECOND;
+ assertSpans(tokenEnd, input, new Span(0, 57));
+ }
+
+ @Test
+ void mapPositionsToSpansHandlesNoPositions() {
+ java.util.List<Double> probs = new java.util.ArrayList<>();
+ Assertions.assertEquals(0,
+ tokenEnd.mapPositionsToSpans(" \t ", new int[0], probs).length);
+ Assertions.assertTrue(probs.isEmpty());
+ Span[] whole = tokenEnd.mapPositionsToSpans(" some text ", new int[0],
probs);
+ Assertions.assertEquals(1, whole.length);
+ Assertions.assertEquals(new Span(2, 11).getStart(), whole[0].getStart());
+ Assertions.assertEquals(new Span(2, 11).getEnd(), whole[0].getEnd());
+ Assertions.assertEquals(java.util.List.of(1d), probs);
+ }
+
+ @Test
+ void mapPositionsToSpansDropsAWhitespaceOnlyCandidateWithItsProbability() {
+ // The whitespace-only-candidate branch is not reachable through
sentPosDetect's position
+ // invariants, but the mapping guarantees alignment structurally: the span
and its
+ // probability are dropped together. The old implementation removed the
probability by index
+ // from a pre-sized array, which could misalign the pairing.
+ java.util.List<Double> probs = new
java.util.ArrayList<>(java.util.List.of(0.9d, 0.8d));
+ Span[] spans = tokenEnd.mapPositionsToSpans("ab cd", new int[] {4, 4},
probs);
+ Assertions.assertEquals(2, spans.length);
+ Assertions.assertEquals(0, spans[0].getStart());
+ Assertions.assertEquals(2, spans[0].getEnd());
+ Assertions.assertEquals(0.9d, spans[0].getProb());
+ Assertions.assertEquals(4, spans[1].getStart());
+ Assertions.assertEquals(6, spans[1].getEnd());
+ Assertions.assertEquals(1d, spans[1].getProb());
+ Assertions.assertEquals(java.util.List.of(0.9d, 1d), probs);
+ }
+
+ @Test
+ void mapPositionsToSpansTrimsTheFullUnicodeWhitespaceSet() {
+ java.util.List<Double> probs = new
java.util.ArrayList<>(java.util.List.of(0.7d));
+ Span[] spans = tokenEnd.mapPositionsToSpans(
+ "One.\u0085\u00A0\u2028Two", new int[] {7}, probs);
+ Assertions.assertEquals(2, spans.length);
+ Assertions.assertEquals(0, spans[0].getStart());
+ Assertions.assertEquals(4, spans[0].getEnd());
+ Assertions.assertEquals(7, spans[1].getStart());
+ Assertions.assertEquals(10, spans[1].getEnd());
+ }
+
+ @Test
+ void informationSeparatorsAreContentNotWhitespace() {
+ // Deliberate delta from the old StringUtil-based mapping: the
U+001C..U+001F information
+ // separators are not Unicode White_Space, so they are no longer trimmed
from span edges.
+ java.util.List<Double> probs = new
java.util.ArrayList<>(java.util.List.of(0.7d));
+ Span[] spans = tokenEnd.mapPositionsToSpans("A.\u001C B", new int[] {4},
probs);
+ Assertions.assertEquals(2, spans.length);
+ Assertions.assertEquals(0, spans[0].getStart());
+ Assertions.assertEquals(3, spans[0].getEnd()); // includes the separator
control
+ }
+
+ @Test
+ void mapPositionsToSpansClearsStaleProbsInTheZeroPositionsBranch() {
+ // The probs contract holds in every branch: with no positions, entries a
caller left in the
+ // list are cleared so the list mirrors the returned spans instead of
keeping stale values.
+ List<Double> stale = new ArrayList<>(List.of(0.4d, 0.3d));
+ Span[] spans = tokenEnd.mapPositionsToSpans(" some text ", new int[0],
stale);
+ Assertions.assertEquals(1, spans.length);
+ Assertions.assertEquals(List.of(1d), stale);
+
+ // Blank input: no spans, and the stale entries are gone as well.
+ List<Double> blankStale = new ArrayList<>(List.of(0.4d));
+ Assertions.assertEquals(0,
+ tokenEnd.mapPositionsToSpans(" \t ", new int[0], blankStale).length);
+ Assertions.assertTrue(blankStale.isEmpty());
+ }
+
+ @Test
+ void mapPositionsToSpansAttachesProbabilityOneInTheZeroPositionsBranch() {
+ // Every returned span carries its probability; the whole-text span of the
zero-positions
+ // branch is no exception (it used to report the Span default of 0.0 via
getProb()).
+ List<Double> probs = new ArrayList<>();
+ Span[] spans = tokenEnd.mapPositionsToSpans("no end of sentence", new
int[0], probs);
+ Assertions.assertEquals(1, spans.length);
+ Assertions.assertEquals(1d, spans[0].getProb());
+ Assertions.assertEquals(List.of(1d), probs);
+ }
+
+ @Test
+ void controlOnlyInputIsOneSpanOfContent() {
+ // Deliberate delta, the whole-text counterpart of
+ // informationSeparatorsAreContentNotWhitespace: information separators
are content, so
+ // control-only input no longer trims to nothing. The old mapping returned
no spans here.
+ List<Double> probs = new ArrayList<>();
+ Span[] spans = tokenEnd.mapPositionsToSpans("\u001C\u001C", new int[0],
probs);
+ Assertions.assertEquals(1, spans.length);
+ Assertions.assertEquals(0, spans[0].getStart());
+ Assertions.assertEquals(2, spans[0].getEnd());
+ Assertions.assertEquals(1d, spans[0].getProb());
+ Assertions.assertEquals(List.of(1d), probs);
+
+ // The same input through the public API: no end-of-sentence characters,
so the detector
+ // takes the zero-positions branch and reports the control characters as
one sentence span.
+ assertSpans(tokenEnd, "\u001C\u001C", new Span(0, 2));
+ Assertions.assertArrayEquals(new double[] {1d}, tokenEnd.probs());
+ }
+
+ @Test
+ void mapPositionsToSpansPreservesSupplementaryCharactersAtSpanEdges() {
+ // The trimming cursors scan by code point (CharClass discipline).
Surrogate pairs at span
+ // edges must survive intact; U+1D518 and U+1D51E are supplementary-plane
letters.
+ String frakturU = "\uD835\uDD18"; // MATHEMATICAL FRAKTUR CAPITAL U
+ String frakturA = "\uD835\uDD1E"; // MATHEMATICAL FRAKTUR SMALL A
+
+ List<Double> probs = new ArrayList<>();
+ String noPositions = " " + frakturU + "no end" + frakturA + " ";
+ Span[] spans = tokenEnd.mapPositionsToSpans(noPositions, new int[0],
probs);
+ Assertions.assertEquals(1, spans.length);
+ Assertions.assertEquals(frakturU + "no end" + frakturA,
+ spans[0].getCoveredText(noPositions).toString());
+
+ // A position between two sentences that start and end with supplementary
characters.
+ probs = new ArrayList<>(List.of(0.9d));
+ String twoParts = frakturU + "one. " + frakturA + "two";
+ Span[] parts = tokenEnd.mapPositionsToSpans(twoParts, new int[] {7},
probs);
+ Assertions.assertEquals(2, parts.length);
+ Assertions.assertEquals(frakturU + "one.",
parts[0].getCoveredText(twoParts).toString());
+ Assertions.assertEquals(frakturA + "two",
parts[1].getCoveredText(twoParts).toString());
+ Assertions.assertEquals(List.of(0.9d, 1d), probs);
+ }
+
+ @Test
+ void supplementaryCharactersFlowThroughTheDetectorUnharmed() {
+ // End-to-end through the trained model: the whitespace cursors of the
detection loop also
+ // scan by code point, so a surrogate pair adjacent to the sentence
boundary stays intact.
+ String fraktur = "\uD835\uDD18\uD835\uDD1E"; // two supplementary letters
as a word
+ String input = "This is a " + fraktur + " test. " + SECOND;
+ Span[] spans = tokenEnd.sentPosDetect(input);
+ Assertions.assertEquals(2, spans.length);
+ Assertions.assertEquals("This is a " + fraktur + " test.",
+ spans[0].getCoveredText(input).toString());
+ Assertions.assertEquals(SECOND, spans[1].getCoveredText(input).toString());
+ }
+
+ @Test
+ void informationSeparatorGluedToTheDelimiterExtendsTheTokenEndBoundary() {
+ // Detection-loop delta of the Unicode White_Space set (OPENNLP-205),
pinned deliberately:
+ // U+001C directly after the delimiter is content now. With useTokenEnd
the token containing
+ // the end-of-sentence character runs through "\u001CThere" up to the next
real whitespace,
+ // and the second sentence starts after that token; without useTokenEnd
the second sentence
+ // starts at the separator and carries it as leading content. The old
StringUtil-based
+ // cursors treated U+001C as whitespace and started the second sentence
directly behind it
+ // in both configurations. Feature generation is untouched; only the
placement of the
+ // accepted position moved.
+ String input = "This is a test.\u001C" + SECOND;
+ assertSpans(tokenEnd, input, new Span(0, 21), new Span(22, 57));
+ assertSpans(noTokenEnd, input, new Span(0, 15), new Span(15, 57));
+ }
+
+ @Test
+ void
informationSeparatorInsideAbbreviationLikeTokenChangesTheCandidateSkip() {
+ // Detection-loop delta, pinned deliberately: in "z.\u001Cb. x" the first
delimiter sits in
+ // front of non-whitespace content (U+001C is content now), so the
delimiter-run skip
+ // heuristic merges it into the multi-period token and only the second
delimiter is scored.
+ // The old cursors saw U+001C as whitespace and scored the first delimiter
as well. The
+ // model splits at the second delimiter, so the multi-period token stays
one sentence and
+ // "x" becomes the next; the separator is covered as content inside the
first span.
+ String input = "z.\u001Cb. x";
+ assertSpans(tokenEnd, input, new Span(0, 5), new Span(6, 7));
+ assertSpans(noTokenEnd, input, new Span(0, 5), new Span(6, 7));
+ }
+}
