This is an automated email from the ASF dual-hosted git repository. krickert pushed a commit to branch OPENNLP-1894-lattice-cjk in repository https://gitbox.apache.org/repos/asf/opennlp.git
commit b48a8eb38b21aa00b16b2c0324d89982ffcd1940 Author: Kristian Rickert <[email protected]> AuthorDate: Fri Jul 17 00:53:57 2026 -0400 OPENNLP-1894: Precompute category runs, unbox the trie, and reject inexpressible dictionary values The lattice tokenizer rescanned the same-category run from every position, so a run of L characters cost on the order of L squared category lookups; a 16,000-character katakana run measured around half a second. One right-to-left pass per stretch now fixes every position's category and run end, and the same 16,000-character run tokenizes in about half a millisecond at 31 million characters per second. The character table holds Category instances instead of names, so the per-character path compares by identity with no name-map lookup, and a char.def mapping to an undefined category now fails at load naming the code point. The lexicon trie's children are sorted character arrays found by binary search, so a descent no longer boxes a Character per step. matrix.def loading rejects connection costs outside the 16-bit range instead of silently truncating them, and dimension products beyond the addressable array size fail at the header. The unigram segmenter's unknown-character fallback advances one code point, never one code unit, so an unknown supplementary character is stepped over whole and no span can split its surrogate halves. --- .../tools/tokenize/lattice/LatticeTokenizer.java | 66 ++++++++-- .../tools/tokenize/lattice/MecabDictionary.java | 141 +++++++++++++++++---- .../tools/tokenize/lattice/UnigramSegmenter.java | 11 +- .../tokenize/lattice/LatticeTokenizerTest.java | 138 ++++++++++++++++++++ .../tokenize/lattice/UnigramSegmenterTest.java | 24 ++++ 5 files changed, 337 insertions(+), 43 deletions(-) diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/LatticeTokenizer.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/LatticeTokenizer.java index 70d9c774f..f7a479bf8 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/LatticeTokenizer.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/LatticeTokenizer.java @@ -143,11 +143,19 @@ public class LatticeTokenizer implements Tokenizer { final boolean[] reachable = new boolean[length + 1]; reachable[0] = true; + // One right-to-left pass fixes each position's category and same-category run end, + // so candidate generation reads them instead of rescanning the run from every + // position, which would cost the square of the run length on long uniform runs. + final Category[] categoryAt = new Category[length]; + final int[] runEndAt = new int[length]; + computeCategoryRuns(text, from, to, categoryAt, runEndAt); + for (int i = 0; i < length; i++) { if (!reachable[i]) { continue; } - final List<Node> candidates = candidates(text, from, to, i); + final List<Node> candidates = + candidates(text, from, to, i, categoryAt[i], runEndAt[i]); for (final Node candidate : candidates) { relax(candidate, i == 0 ? null : endingAt.get(i)); if (candidate.pathCost < Long.MAX_VALUE) { @@ -199,8 +207,39 @@ public class LatticeTokenizer implements Tokenizer { } } + /** + * Fills the per-position category and same-category run end for one stretch, in one + * right-to-left pass over its code points. Positions inside a surrogate pair keep a + * {@code null} category; no candidate ever starts there. + * + * @param text The text being segmented. + * @param from The stretch start. + * @param to The exclusive stretch end. + * @param categoryAt Receives each position's category, indexed by {@code + * position - from}. + * @param runEndAt Receives each position's exclusive same-category run end, indexed + * the same way. + */ + private void computeCategoryRuns(String text, int from, int to, + Category[] categoryAt, int[] runEndAt) { + int next = -1; + for (int position = to; position > from; ) { + final int codePoint = text.codePointBefore(position); + position -= Character.charCount(codePoint); + final int index = position - from; + categoryAt[index] = dictionary.categoryOf(codePoint); + if (next >= 0 && categoryAt[next] == categoryAt[index]) { + runEndAt[index] = runEndAt[next]; + } else { + runEndAt[index] = next >= 0 ? next + from : to; + } + next = index; + } + } + /** Gathers lexicon matches and unknown-word candidates starting at one position. */ - private List<Node> candidates(String text, int from, int to, int offset) { + private List<Node> candidates(String text, int from, int to, int offset, + Category positionCategory, int positionRunEnd) { final int position = from + offset; final List<Node> candidates = new ArrayList<>(); final boolean[] matched = new boolean[1]; @@ -213,19 +252,22 @@ public class LatticeTokenizer implements Tokenizer { final boolean lexiconMatch = matched[0]; final int codePoint = text.codePointAt(position); - final Category category = dictionary.categoryOf(codePoint); + final Category category; + final int runEnd; + if (positionCategory == null) { + // Only a lexicon surface ending inside a surrogate pair could make such a + // position reachable; classify the stray code unit on the spot so the lattice + // stays connected the way it always did. + category = dictionary.categoryOf(codePoint); + runEnd = position + Character.charCount(codePoint); + } else { + category = positionCategory; + runEnd = positionRunEnd; + } if (!lexiconMatch || category.invoke()) { - int run = position + Character.charCount(codePoint); - while (run < to) { - final int next = text.codePointAt(run); - if (!dictionary.categoryOf(next).name().equals(category.name())) { - break; - } - run += Character.charCount(next); - } final List<WordEntry> templates = dictionary.unknownEntries(category.name()); if (templates != null) { - addUnknown(candidates, text, position, run, category, templates); + addUnknown(candidates, text, position, runEnd, category, templates); } } if (candidates.isEmpty()) { diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionary.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionary.java index c8138d7cf..4a34e24d2 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionary.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionary.java @@ -58,10 +58,56 @@ public final class MecabDictionary { record Category(String name, boolean invoke, boolean group, int length) { } - /** One node of the lexicon trie, keyed by the next surface character. */ + /** + * One frozen node of the lexicon trie: children are held as a sorted character + * array with a parallel node array and found by binary search, so a descent never + * boxes the character the way a map keyed by {@link Character} would on every step + * of the innermost matching loop. + */ private static final class TrieNode { - private final Map<Character, TrieNode> children = new HashMap<>(); + + private final char[] keys; + private final TrieNode[] nodes; + private final List<WordEntry> entries; + + private TrieNode(char[] keys, TrieNode[] nodes, List<WordEntry> entries) { + this.keys = keys; + this.nodes = nodes; + this.entries = entries; + } + + /** + * Descends one character. + * + * @param c The next surface character. + * @return The child node, or {@code null} when no surface continues with {@code c}. + */ + private TrieNode child(char c) { + final int index = Arrays.binarySearch(keys, c); + return index >= 0 ? nodes[index] : null; + } + } + + /** One mutable trie node during construction, frozen into a {@link TrieNode}. */ + private static final class TrieBuilderNode { + + private final Map<Character, TrieBuilderNode> children = new HashMap<>(); private List<WordEntry> entries; + + /** Freezes this node and its subtree into the sorted-array form. */ + private TrieNode freeze() { + final char[] keys = new char[children.size()]; + int i = 0; + for (final Character key : children.keySet()) { + keys[i++] = key; + } + Arrays.sort(keys); + final TrieNode[] nodes = new TrieNode[keys.length]; + for (int k = 0; k < keys.length; k++) { + nodes[k] = children.get(keys[k]).freeze(); + } + return new TrieNode(keys, nodes, entries); + } } /** @@ -77,13 +123,13 @@ public final class MecabDictionary { */ private static final class CategoryTable { - private final String[] bmp; + private final Category[] bmp; private final int[] rangeStart; private final int[] rangeEnd; - private final String[] rangeCategory; + private final Category[] rangeCategory; - private CategoryTable(String[] bmp, int[] rangeStart, int[] rangeEnd, - String[] rangeCategory) { + private CategoryTable(Category[] bmp, int[] rangeStart, int[] rangeEnd, + Category[] rangeCategory) { this.bmp = bmp; this.rangeStart = rangeStart; this.rangeEnd = rangeEnd; @@ -91,12 +137,16 @@ public final class MecabDictionary { } /** - * Looks up the category name a {@code char.def} mapping gives a code point. + * Looks up the category a {@code char.def} mapping gives a code point. The table + * holds the {@link Category} instances themselves, so a lookup on the tokenizer's + * per-character path costs one array read or one binary search, never a name map + * access, and two code points of one category share one instance to compare by + * identity. * * @param codePoint The code point to classify. - * @return The category name, or {@code null} when no mapping covers the code point. + * @return The category, or {@code null} when no mapping covers the code point. */ - private String categoryOf(int codePoint) { + private Category categoryOf(int codePoint) { if (codePoint <= Character.MAX_VALUE) { return bmp[codePoint]; } @@ -149,7 +199,7 @@ public final class MecabDictionary { * * @return The table. Never {@code null}. */ - private CategoryTable build() { + private CategoryTable build(Map<String, Category> categories) throws IOException { // Cut the supplementary ranges at every boundary they introduce, so that each // resulting elementary interval is covered by a single winning range and the // table stays sorted and non-overlapping for binary search. @@ -160,7 +210,7 @@ public final class MecabDictionary { } Arrays.sort(edges); final List<int[]> intervals = new ArrayList<>(); - final List<String> categories = new ArrayList<>(); + final List<String> winners = new ArrayList<>(); for (int i = 0; i < edges.length - 1; i++) { if (edges[i] == edges[i + 1]) { continue; @@ -171,11 +221,11 @@ public final class MecabDictionary { } final int previous = intervals.size() - 1; if (previous >= 0 && intervals.get(previous)[1] == edges[i] - 1 - && categories.get(previous).equals(winner)) { + && winners.get(previous).equals(winner)) { intervals.get(previous)[1] = edges[i + 1] - 1; } else { intervals.add(new int[] {edges[i], edges[i + 1] - 1}); - categories.add(winner); + winners.add(winner); } } final int[] starts = new int[intervals.size()]; @@ -184,7 +234,32 @@ public final class MecabDictionary { starts[i] = intervals.get(i)[0]; ends[i] = intervals.get(i)[1]; } - return new CategoryTable(bmp, starts, ends, categories.toArray(new String[0])); + final Category[] resolvedBmp = new Category[bmp.length]; + for (int c = 0; c < bmp.length; c++) { + if (bmp[c] != null) { + resolvedBmp[c] = resolve(bmp[c], categories, c); + } + } + final Category[] resolvedRanges = new Category[winners.size()]; + for (int i = 0; i < winners.size(); i++) { + resolvedRanges[i] = resolve(winners.get(i), categories, starts[i]); + } + return new CategoryTable(resolvedBmp, starts, ends, resolvedRanges); + } + + /** + * Resolves a mapped category name against the defined categories, so a mapping to + * a name the {@code char.def} category section never defined fails at load with + * the offending code point instead of falling back silently at lookup time. + */ + private static Category resolve(String name, Map<String, Category> categories, + int codePoint) throws IOException { + final Category category = categories.get(name); + if (category == null) { + throw new IOException(String.format( + "char.def maps U+%04X to the undefined category %s", codePoint, name)); + } + return category; } /** @@ -222,6 +297,7 @@ public final class MecabDictionary { private final int rightSize; private final Map<String, Category> categories; private final CategoryTable categoryTable; + private final Category defaultCategory; private final Map<String, List<WordEntry>> unknownEntries; private MecabDictionary(TrieNode lexicon, int maxSurfaceLength, @@ -233,6 +309,7 @@ public final class MecabDictionary { this.rightSize = rightSize; this.categories = categories; this.categoryTable = categoryTable; + this.defaultCategory = categories.get("DEFAULT"); this.unknownEntries = unknownEntries; } @@ -282,7 +359,12 @@ public final class MecabDictionary { throw new IOException("matrix.def dimensions must be positive, got " + leftSize + " " + rightSize); } - final short[] costs = new short[leftSize * rightSize]; + final long cells = (long) leftSize * rightSize; + if (cells > Integer.MAX_VALUE) { + throw new IOException("matrix.def dimensions " + leftSize + " x " + rightSize + + " overflow the addressable connection matrix"); + } + final short[] costs = new short[(int) cells]; for (int i = 1; i < matrixLines.size(); i++) { final String line = matrixLines.get(i).trim(); if (line.isEmpty()) { @@ -299,7 +381,13 @@ public final class MecabDictionary { + ": context ids " + right + " " + left + " are outside the declared dimensions " + leftSize + " " + rightSize); } - costs[right * rightSize + left] = (short) parseInt(fields[2], "matrix.def", i + 1); + final int cost = parseInt(fields[2], "matrix.def", i + 1); + if (cost < Short.MIN_VALUE || cost > Short.MAX_VALUE) { + throw new IOException("malformed matrix.def line " + (i + 1) + + ": connection cost " + cost + " is outside the 16-bit range the" + + " format defines"); + } + costs[right * rightSize + left] = (short) cost; } final Map<String, List<WordEntry>> lexicon = new HashMap<>(); @@ -318,26 +406,26 @@ public final class MecabDictionary { final CategoryTableBuilder categoryTable = new CategoryTableBuilder(); readCharacterDefinition(directory.resolve("char.def"), charset, categories, categoryTable); - final Map<String, List<WordEntry>> unknown = new HashMap<>(); readLexicon(directory.resolve("unk.def"), charset, unknown, leftSize, rightSize); return new MecabDictionary(buildTrie(lexicon), maxSurface, costs, rightSize, - categories, categoryTable.build(), unknown); + categories, categoryTable.build(categories), unknown); } /** Folds the surface-keyed lexicon into a character trie for prefix search. */ private static TrieNode buildTrie(Map<String, List<WordEntry>> lexicon) { - final TrieNode root = new TrieNode(); + final TrieBuilderNode root = new TrieBuilderNode(); for (final Map.Entry<String, List<WordEntry>> entry : lexicon.entrySet()) { - TrieNode node = root; + TrieBuilderNode node = root; final String surface = entry.getKey(); for (int i = 0; i < surface.length(); i++) { - node = node.children.computeIfAbsent(surface.charAt(i), key -> new TrieNode()); + node = node.children.computeIfAbsent(surface.charAt(i), + key -> new TrieBuilderNode()); } node.entries = List.copyOf(entry.getValue()); } - return root; + return root.freeze(); } /** @@ -448,7 +536,7 @@ public final class MecabDictionary { List<WordEntry> lookup(String surface) { TrieNode node = lexicon; for (int i = 0; i < surface.length() && node != null; i++) { - node = node.children.get(surface.charAt(i)); + node = node.child(surface.charAt(i)); } return node == null ? null : node.entries; } @@ -465,7 +553,7 @@ public final class MecabDictionary { void prefixMatches(String text, int from, int to, PrefixMatchConsumer consumer) { TrieNode node = lexicon; for (int i = from; i < to; i++) { - node = node.children.get(text.charAt(i)); + node = node.child(text.charAt(i)); if (node == null) { return; } @@ -501,9 +589,8 @@ public final class MecabDictionary { * mapping covers the code point. Never {@code null}. */ Category categoryOf(int codePoint) { - final String name = categoryTable.categoryOf(codePoint); - final Category category = name == null ? null : categories.get(name); - return category != null ? category : categories.get("DEFAULT"); + final Category category = categoryTable.categoryOf(codePoint); + return category != null ? category : defaultCategory; } /** diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/UnigramSegmenter.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/UnigramSegmenter.java index ea3d188bc..b4cfaa8ea 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/UnigramSegmenter.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/UnigramSegmenter.java @@ -218,11 +218,14 @@ public class UnigramSegmenter implements Tokenizer { continue; } // A single-character step at the unknown log-probability keeps every position - // reachable even where no lexicon word matches. + // reachable even where no lexicon word matches. The step advances one code + // point, never one code unit, so an unknown supplementary character is stepped + // over whole and no span boundary can land between its surrogate halves. + final int width = Character.charCount(text.codePointAt(from + i)); final double fallback = best[i] + unknownLogProbability; - if (fallback > best[i + 1]) { - best[i + 1] = fallback; - previous[i + 1] = i; + if (i + width <= length && fallback > best[i + width]) { + best[i + width] = fallback; + previous[i + width] = i; } WordTrie node = trie; for (int j = from + i; j < to; j++) { diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeTokenizerTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeTokenizerTest.java index bf7a5f8c3..e1be5a4a9 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeTokenizerTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeTokenizerTest.java @@ -484,4 +484,142 @@ public class LatticeTokenizerTest { () -> MecabDictionary.load(null)); Assertions.assertThrows(IllegalArgumentException.class, () -> tokenizer.analyze(null)); } + + /** + * Verifies the supplementary range table's interval cutting and precedence: a later + * {@code char.def} mapping strictly inside an earlier one wins exactly on its own + * stretch, and the earlier category resumes after it, so the cut produces three + * intervals from two overlapping ranges. + */ + @Test + void testLaterSupplementaryMappingWinsInsideAnEarlierRange(@TempDir Path overlapped) + throws IOException { + Files.write(overlapped.resolve("lexicon.csv"), + "\u6771,0,0,6000,noun\n".getBytes(StandardCharsets.UTF_8)); + Files.write(overlapped.resolve("matrix.def"), + "1 1\n0 0 0\n".getBytes(StandardCharsets.UTF_8)); + Files.write(overlapped.resolve("char.def"), String.join("\n", + "DEFAULT 0 1 0", + "KANJI 0 0 2", + "LATIN 1 1 0", + "", + "0x20000..0x2FFFF KANJI", + "0x24000..0x25000 LATIN", + "").getBytes(StandardCharsets.UTF_8)); + Files.write(overlapped.resolve("unk.def"), + "DEFAULT,0,0,10000,symbol,unknown\n".getBytes(StandardCharsets.UTF_8)); + + final MecabDictionary dictionary = MecabDictionary.load(overlapped); + Assertions.assertEquals("KANJI", dictionary.categoryOf(0x20000).name()); + Assertions.assertEquals("KANJI", dictionary.categoryOf(0x23FFF).name()); + Assertions.assertEquals("LATIN", dictionary.categoryOf(0x24000).name()); + Assertions.assertEquals("LATIN", dictionary.categoryOf(0x25000).name()); + Assertions.assertEquals("KANJI", dictionary.categoryOf(0x25001).name()); + Assertions.assertEquals("KANJI", dictionary.categoryOf(0x2FFFF).name()); + Assertions.assertEquals("DEFAULT", dictionary.categoryOf(0x30000).name()); + } + + /** + * Verifies a {@code char.def} range straddling the BMP boundary: the part up to + * U+FFFF lands in the directly indexed table and the rest in the range table, and + * both halves answer the same category with no gap at the seam. + */ + @Test + void testCharDefRangeStraddlingTheBmpBoundary(@TempDir Path straddling) + throws IOException { + Files.write(straddling.resolve("lexicon.csv"), + "\u6771,0,0,6000,noun\n".getBytes(StandardCharsets.UTF_8)); + Files.write(straddling.resolve("matrix.def"), + "1 1\n0 0 0\n".getBytes(StandardCharsets.UTF_8)); + Files.write(straddling.resolve("char.def"), String.join("\n", + "DEFAULT 0 1 0", + "LATIN 1 1 0", + "", + "0xFF00..0x10040 LATIN", + "").getBytes(StandardCharsets.UTF_8)); + Files.write(straddling.resolve("unk.def"), + "DEFAULT,0,0,10000,symbol,unknown\n".getBytes(StandardCharsets.UTF_8)); + + final MecabDictionary dictionary = MecabDictionary.load(straddling); + Assertions.assertEquals("LATIN", dictionary.categoryOf(0xFF00).name()); + Assertions.assertEquals("LATIN", dictionary.categoryOf(0xFFFF).name()); + Assertions.assertEquals("LATIN", dictionary.categoryOf(0x10000).name()); + Assertions.assertEquals("LATIN", dictionary.categoryOf(0x10040).name()); + Assertions.assertEquals("DEFAULT", dictionary.categoryOf(0x10041).name()); + Assertions.assertEquals("DEFAULT", dictionary.categoryOf(0xFEFF).name()); + } + + /** + * Verifies that a {@code char.def} mapping to a category its category section never + * defined fails at load, naming the code point and the ghost category, instead of + * silently falling back to DEFAULT at lookup time. + */ + @Test + void testMappingToUndefinedCategoryFailsLoud(@TempDir Path ghost) throws IOException { + Files.write(ghost.resolve("lexicon.csv"), + "\u6771,0,0,6000,noun\n".getBytes(StandardCharsets.UTF_8)); + Files.write(ghost.resolve("matrix.def"), + "1 1\n0 0 0\n".getBytes(StandardCharsets.UTF_8)); + Files.write(ghost.resolve("char.def"), String.join("\n", + "DEFAULT 0 1 0", + "", + "0x0100..0x0110 GHOST", + "").getBytes(StandardCharsets.UTF_8)); + Files.write(ghost.resolve("unk.def"), + "DEFAULT,0,0,10000,symbol,unknown\n".getBytes(StandardCharsets.UTF_8)); + + final IOException e = Assertions.assertThrows(IOException.class, + () -> MecabDictionary.load(ghost)); + Assertions.assertEquals("char.def maps U+0100 to the undefined category GHOST", + e.getMessage()); + } + + /** + * Verifies that a connection cost outside the 16-bit range the binary matrix format + * defines is rejected at load instead of being truncated by the narrowing cast into + * a silently different cost. + */ + @Test + void testMatrixCostOutsideShortRangeFailsLoud(@TempDir Path broken) throws IOException { + writeUnitMatrixDictionary(broken); + Files.write(broken.resolve("matrix.def"), + "1 1\n0 0 40000\n".getBytes(StandardCharsets.UTF_8)); + final IOException e = Assertions.assertThrows(IOException.class, + () -> MecabDictionary.load(broken)); + Assertions.assertEquals("malformed matrix.def line 2: connection cost 40000 is" + + " outside the 16-bit range the format defines", e.getMessage()); + } + + /** + * Verifies that {@code matrix.def} dimensions whose product exceeds the addressable + * array size fail loud at the header instead of overflowing the int multiplication + * into a negative or wrapped allocation size. + */ + @Test + void testMatrixDimensionProductBeyondIntRangeFailsLoud(@TempDir Path broken) + throws IOException { + writeUnitMatrixDictionary(broken); + Files.write(broken.resolve("matrix.def"), + "70000 70000\n".getBytes(StandardCharsets.UTF_8)); + final IOException e = Assertions.assertThrows(IOException.class, + () -> MecabDictionary.load(broken)); + Assertions.assertEquals("matrix.def dimensions 70000 x 70000 overflow the" + + " addressable connection matrix", e.getMessage()); + } + + /** + * Verifies that a {@code matrix.def} data row naming context ids outside the + * declared dimensions is rejected at load with the offending line and ids. + */ + @Test + void testMatrixRowContextIdsOutsideDimensionsFailLoud(@TempDir Path broken) + throws IOException { + writeUnitMatrixDictionary(broken); + Files.write(broken.resolve("matrix.def"), + "1 1\n2 0 5\n".getBytes(StandardCharsets.UTF_8)); + final IOException e = Assertions.assertThrows(IOException.class, + () -> MecabDictionary.load(broken)); + Assertions.assertEquals("malformed matrix.def line 2: context ids 2 0 are outside" + + " the declared dimensions 1 1", e.getMessage()); + } } diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/UnigramSegmenterTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/UnigramSegmenterTest.java index 9725b8c72..e6ebd604e 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/UnigramSegmenterTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/UnigramSegmenterTest.java @@ -176,4 +176,28 @@ public class UnigramSegmenterTest { Assertions.assertThrows(IllegalArgumentException.class, () -> segmenter.tokenizePos(null)); } + + /** + * Verifies that the unknown-character fallback advances one code point, never one + * code unit: a supplementary character absent from the lexicon comes back as one + * span over both of its surrogate halves, and no span boundary ever lands between + * them. + */ + @org.junit.jupiter.api.Test + void testUnknownSupplementaryCharacterIsNeverSplit() { + // U+20BB7, a CJK extension B ideograph, written as its surrogate pair + final String text = "\uD842\uDFB7\uD842\uDFB7"; + final opennlp.tools.util.Span[] spans = segmenter.tokenizePos(text); + for (final opennlp.tools.util.Span span : spans) { + Assertions.assertEquals(0, span.getStart() % 2, + "span must start on a code point boundary: " + span); + Assertions.assertEquals(0, span.getEnd() % 2, + "span must end on a code point boundary: " + span); + } + int covered = 0; + for (final opennlp.tools.util.Span span : spans) { + covered += span.length(); + } + Assertions.assertEquals(text.length(), covered); + } }
