This is an automated email from the ASF dual-hosted git repository. krickert pushed a commit to branch OPENNLP-1894-lattice-cjk in repository https://gitbox.apache.org/repos/asf/opennlp.git
commit 4d76e948d4ab4381bd55f38ef46b60ec68dbe19b Author: Kristian Rickert <[email protected]> AuthorDate: Thu Jul 16 22:09:30 2026 -0400 OPENNLP-1894: Categorize by code point, keep unknown candidates inside their category run, and validate context ids at load --- .../tools/tokenize/lattice/LatticeTokenizer.java | 47 +++- .../tools/tokenize/lattice/MecabDictionary.java | 263 ++++++++++++++++++--- .../tokenize/lattice/LatticeTokenizerTest.java | 181 ++++++++++++++ 3 files changed, 442 insertions(+), 49 deletions(-) diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/LatticeTokenizer.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/LatticeTokenizer.java index cb610f74f..70d9c774f 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/LatticeTokenizer.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/LatticeTokenizer.java @@ -212,16 +212,20 @@ public class LatticeTokenizer implements Tokenizer { }); final boolean lexiconMatch = matched[0]; - final Category category = dictionary.categoryOf(text.charAt(position)); + final int codePoint = text.codePointAt(position); + final Category category = dictionary.categoryOf(codePoint); if (!lexiconMatch || category.invoke()) { - int run = position + 1; - while (run < to - && dictionary.categoryOf(text.charAt(run)).name().equals(category.name())) { - run++; + int run = position + Character.charCount(codePoint); + while (run < to) { + final int next = text.codePointAt(run); + if (!dictionary.categoryOf(next).name().equals(category.name())) { + break; + } + run += Character.charCount(next); } final List<WordEntry> templates = dictionary.unknownEntries(category.name()); if (templates != null) { - addUnknown(candidates, position, run, to, category, templates); + addUnknown(candidates, text, position, run, category, templates); } } if (candidates.isEmpty()) { @@ -230,7 +234,8 @@ public class LatticeTokenizer implements Tokenizer { final List<WordEntry> fallback = dictionary.unknownEntries("DEFAULT"); if (fallback != null) { for (final WordEntry entry : fallback) { - candidates.add(new Node(position, position + 1, entry, true)); + candidates.add( + new Node(position, position + Character.charCount(codePoint), entry, true)); } } } @@ -241,22 +246,38 @@ public class LatticeTokenizer implements Tokenizer { return candidates; } - /** Emits unknown-word candidates per the category's grouping and length settings. */ - private static void addUnknown(List<Node> candidates, int position, int runEnd, - int to, Category category, List<WordEntry> templates) { + /** + * Emits unknown-word candidates per the category's grouping and length settings. + * + * <p>Every candidate stays inside the same-category run, so an unknown word never + * glues characters of different categories together, and every length counts whole + * characters rather than code units.</p> + * + * @param candidates Receives the candidates. + * @param text The text being segmented. + * @param position The position the candidates start at. + * @param runEnd The exclusive end of the same-category run starting at + * {@code position}. + * @param category The category of that run. + * @param templates The category's unknown-word templates. + */ + private static void addUnknown(List<Node> candidates, String text, int position, + int runEnd, Category category, List<WordEntry> templates) { if (category.group()) { for (final WordEntry entry : templates) { candidates.add(new Node(position, runEnd, entry, true)); } } final int lengths = category.length(); - for (int length = 1; length <= lengths && position + length <= to; length++) { - if (category.group() && position + length == runEnd) { + int end = position; + for (int length = 1; length <= lengths && end < runEnd; length++) { + end += Character.charCount(text.codePointAt(end)); + if (category.group() && end == runEnd) { // This length coincides with the grouped run emitted above; skip the duplicate. continue; } for (final WordEntry entry : templates) { - candidates.add(new Node(position, position + length, entry, true)); + candidates.add(new Node(position, end, entry, true)); } } } diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionary.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionary.java index 908bc75f2..c8138d7cf 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionary.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionary.java @@ -24,6 +24,7 @@ import java.nio.file.DirectoryStream; import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -63,6 +64,146 @@ public final class MecabDictionary { private List<WordEntry> entries; } + /** + * The {@code char.def} code point to category name mapping, over the whole Unicode + * code point range. + * + * <p>The Basic Multilingual Plane is held in a directly indexed array, which is one + * reference per BMP code point and is the entire mapping for a BMP-only dictionary. + * The supplementary planes are held as a sorted, non-overlapping range table searched + * by binary search, because dictionaries map them in a handful of large blocks: a + * directly indexed array over all of Unicode would spend more than four megabytes of + * references to say what a few range rows already say.</p> + */ + private static final class CategoryTable { + + private final String[] bmp; + private final int[] rangeStart; + private final int[] rangeEnd; + private final String[] rangeCategory; + + private CategoryTable(String[] bmp, int[] rangeStart, int[] rangeEnd, + String[] rangeCategory) { + this.bmp = bmp; + this.rangeStart = rangeStart; + this.rangeEnd = rangeEnd; + this.rangeCategory = rangeCategory; + } + + /** + * Looks up the category name a {@code char.def} mapping gives a code point. + * + * @param codePoint The code point to classify. + * @return The category name, or {@code null} when no mapping covers the code point. + */ + private String categoryOf(int codePoint) { + if (codePoint <= Character.MAX_VALUE) { + return bmp[codePoint]; + } + int low = 0; + int high = rangeStart.length - 1; + while (low <= high) { + final int middle = (low + high) >>> 1; + if (codePoint < rangeStart[middle]) { + high = middle - 1; + } else if (codePoint > rangeEnd[middle]) { + low = middle + 1; + } else { + return rangeCategory[middle]; + } + } + return null; + } + } + + /** + * Collects {@code char.def} mappings in file order and folds them into a + * {@link CategoryTable}, giving a later mapping precedence over an earlier one that + * covers the same code point, which is what direct indexing does for the BMP. + */ + private static final class CategoryTableBuilder { + + private final String[] bmp = new String[Character.MAX_VALUE + 1]; + private final List<int[]> bounds = new ArrayList<>(); + private final List<String> names = new ArrayList<>(); + + /** + * Records one inclusive code point range's category. + * + * @param from The first code point of the range. + * @param to The last code point of the range, inclusive. + * @param category The category name to give the range. Must not be {@code null}. + */ + private void map(int from, int to, String category) { + for (int c = from; c <= Math.min(to, Character.MAX_VALUE); c++) { + bmp[c] = category; + } + if (to > Character.MAX_VALUE) { + bounds.add(new int[] {Math.max(from, Character.MAX_VALUE + 1), to}); + names.add(category); + } + } + + /** + * Folds the recorded mappings into their lookup table. + * + * @return The table. Never {@code null}. + */ + private CategoryTable build() { + // Cut the supplementary ranges at every boundary they introduce, so that each + // resulting elementary interval is covered by a single winning range and the + // table stays sorted and non-overlapping for binary search. + final int[] edges = new int[bounds.size() * 2]; + for (int i = 0; i < bounds.size(); i++) { + edges[i * 2] = bounds.get(i)[0]; + edges[i * 2 + 1] = bounds.get(i)[1] + 1; + } + Arrays.sort(edges); + final List<int[]> intervals = new ArrayList<>(); + final List<String> categories = new ArrayList<>(); + for (int i = 0; i < edges.length - 1; i++) { + if (edges[i] == edges[i + 1]) { + continue; + } + final String winner = lastCovering(edges[i]); + if (winner == null) { + continue; + } + final int previous = intervals.size() - 1; + if (previous >= 0 && intervals.get(previous)[1] == edges[i] - 1 + && categories.get(previous).equals(winner)) { + intervals.get(previous)[1] = edges[i + 1] - 1; + } else { + intervals.add(new int[] {edges[i], edges[i + 1] - 1}); + categories.add(winner); + } + } + final int[] starts = new int[intervals.size()]; + final int[] ends = new int[intervals.size()]; + for (int i = 0; i < intervals.size(); i++) { + starts[i] = intervals.get(i)[0]; + ends[i] = intervals.get(i)[1]; + } + return new CategoryTable(bmp, starts, ends, categories.toArray(new String[0])); + } + + /** + * Finds the category of the last recorded range covering a code point. + * + * @param codePoint The code point to look up. + * @return The category name, or {@code null} when no recorded range covers it. + */ + private String lastCovering(int codePoint) { + for (int i = bounds.size() - 1; i >= 0; i--) { + final int[] range = bounds.get(i); + if (codePoint >= range[0] && codePoint <= range[1]) { + return names.get(i); + } + } + return null; + } + } + /** Receives one common-prefix match during {@link #prefixMatches}. */ interface PrefixMatchConsumer { @@ -80,18 +221,18 @@ public final class MecabDictionary { private final short[] connectionCosts; private final int rightSize; private final Map<String, Category> categories; - private final String[] categoryOfChar; + private final CategoryTable categoryTable; private final Map<String, List<WordEntry>> unknownEntries; private MecabDictionary(TrieNode lexicon, int maxSurfaceLength, short[] connectionCosts, int rightSize, Map<String, Category> categories, - String[] categoryOfChar, Map<String, List<WordEntry>> unknownEntries) { + CategoryTable categoryTable, Map<String, List<WordEntry>> unknownEntries) { this.lexicon = lexicon; this.maxSurfaceLength = maxSurfaceLength; this.connectionCosts = connectionCosts; this.rightSize = rightSize; this.categories = categories; - this.categoryOfChar = categoryOfChar; + this.categoryTable = categoryTable; this.unknownEntries = unknownEntries; } @@ -116,25 +257,17 @@ public final class MecabDictionary { * @param charset The encoding the distribution uses, for example UTF-8 or EUC-JP. * Must not be {@code null}. * @return The loaded dictionary. Never {@code null}. - * @throws IOException Thrown if reading fails, a required file is missing, or a file - * is malformed. + * @throws IOException Thrown if reading fails, a required file is missing, a file is + * malformed, or a lexicon entry's context ids are outside the + * {@code matrix.def} dimensions. * @throws IllegalArgumentException Thrown if a parameter is {@code null}. */ public static MecabDictionary load(Path directory, Charset charset) throws IOException { if (directory == null || charset == null) { throw new IllegalArgumentException("directory and charset must not be null"); } - final Map<String, List<WordEntry>> lexicon = new HashMap<>(); - int maxSurface = 0; - try (DirectoryStream<Path> csvFiles = Files.newDirectoryStream(directory, "*.csv")) { - for (final Path csv : csvFiles) { - maxSurface = Math.max(maxSurface, readLexicon(csv, charset, lexicon)); - } - } - if (lexicon.isEmpty()) { - throw new IOException("no lexicon entries found under " + directory); - } - + // The connection matrix is read first because its dimensions are what every + // lexicon entry's context ids have to be inside of. final List<String> matrixLines = readLines(directory.resolve("matrix.def"), charset); if (matrixLines.isEmpty()) { throw new IOException("empty matrix.def under " + directory); @@ -145,6 +278,10 @@ public final class MecabDictionary { } final int leftSize = parseInt(header[0], "matrix.def", 1); final int rightSize = parseInt(header[1], "matrix.def", 1); + if (leftSize < 1 || rightSize < 1) { + throw new IOException("matrix.def dimensions must be positive, got " + + leftSize + " " + rightSize); + } final short[] costs = new short[leftSize * rightSize]; for (int i = 1; i < matrixLines.size(); i++) { final String line = matrixLines.get(i).trim(); @@ -157,19 +294,36 @@ public final class MecabDictionary { } final int right = parseInt(fields[0], "matrix.def", i + 1); final int left = parseInt(fields[1], "matrix.def", i + 1); + if (right < 0 || right >= leftSize || left < 0 || left >= rightSize) { + throw new IOException("malformed matrix.def line " + (i + 1) + + ": context ids " + right + " " + left + + " are outside the declared dimensions " + leftSize + " " + rightSize); + } costs[right * rightSize + left] = (short) parseInt(fields[2], "matrix.def", i + 1); } + final Map<String, List<WordEntry>> lexicon = new HashMap<>(); + int maxSurface = 0; + try (DirectoryStream<Path> csvFiles = Files.newDirectoryStream(directory, "*.csv")) { + for (final Path csv : csvFiles) { + maxSurface = Math.max(maxSurface, + readLexicon(csv, charset, lexicon, leftSize, rightSize)); + } + } + if (lexicon.isEmpty()) { + throw new IOException("no lexicon entries found under " + directory); + } + final Map<String, Category> categories = new HashMap<>(); - final String[] categoryOfChar = new String[Character.MAX_VALUE + 1]; + final CategoryTableBuilder categoryTable = new CategoryTableBuilder(); readCharacterDefinition(directory.resolve("char.def"), charset, categories, - categoryOfChar); + categoryTable); final Map<String, List<WordEntry>> unknown = new HashMap<>(); - readLexicon(directory.resolve("unk.def"), charset, unknown); + readLexicon(directory.resolve("unk.def"), charset, unknown, leftSize, rightSize); return new MecabDictionary(buildTrie(lexicon), maxSurface, costs, rightSize, - categories, categoryOfChar, unknown); + categories, categoryTable.build(), unknown); } /** Folds the surface-keyed lexicon into a character trie for prefix search. */ @@ -186,9 +340,24 @@ public final class MecabDictionary { return root; } - /** Reads one lexicon-format CSV file; returns the longest surface seen. */ + /** + * Reads one lexicon-format CSV file, rejecting any entry whose context ids the + * connection matrix cannot be indexed with. + * + * @param file The file to read. + * @param charset The encoding to decode with. + * @param target Receives the entries, keyed by surface form. + * @param leftSize The first {@code matrix.def} dimension, which bounds right context + * ids. + * @param rightSize The second {@code matrix.def} dimension, which bounds left context + * ids. + * @return The length in characters of the longest surface form read. + * @throws IOException Thrown if the file is missing, an entry is malformed, or an + * entry's context id is outside the matrix dimensions. + */ private static int readLexicon(Path file, Charset charset, - Map<String, List<WordEntry>> target) throws IOException { + Map<String, List<WordEntry>> target, int leftSize, int rightSize) + throws IOException { int maxSurface = 0; int lineNumber = 0; for (final String line : readLines(file, charset)) { @@ -204,9 +373,19 @@ public final class MecabDictionary { if (surface.isEmpty()) { continue; } - final WordEntry entry = new WordEntry( - parseInt(fields.get(1), file.toString(), lineNumber), - parseInt(fields.get(2), file.toString(), lineNumber), + final int leftId = parseInt(fields.get(1), file.toString(), lineNumber); + final int rightId = parseInt(fields.get(2), file.toString(), lineNumber); + if (leftId < 0 || leftId >= rightSize) { + throw new IOException("malformed entry at " + file + " line " + lineNumber + + ": left context id " + leftId + " is outside the matrix.def dimensions " + + leftSize + " " + rightSize); + } + if (rightId < 0 || rightId >= leftSize) { + throw new IOException("malformed entry at " + file + " line " + lineNumber + + ": right context id " + rightId + " is outside the matrix.def dimensions " + + leftSize + " " + rightSize); + } + final WordEntry entry = new WordEntry(leftId, rightId, parseInt(fields.get(3), file.toString(), lineNumber), List.copyOf(fields.subList(4, fields.size()))); target.computeIfAbsent(surface, key -> new ArrayList<>(1)).add(entry); @@ -217,7 +396,8 @@ public final class MecabDictionary { /** Reads char.def: category behavior lines and code point mapping lines. */ private static void readCharacterDefinition(Path file, Charset charset, - Map<String, Category> categories, String[] categoryOfChar) throws IOException { + Map<String, Category> categories, CategoryTableBuilder categoryTable) + throws IOException { int lineNumber = 0; for (final String raw : readLines(file, charset)) { lineNumber++; @@ -240,9 +420,11 @@ public final class MecabDictionary { if (fields.length < 2) { throw new IOException("mapping without category at " + file + " line " + lineNumber); } - for (int c = from; c <= to && c <= Character.MAX_VALUE; c++) { - categoryOfChar[c] = fields[1]; + if (from > to) { + throw new IOException("code point range descends at " + file + " line " + + lineNumber); } + categoryTable.map(from, to, fields[1]); } else { if (fields.length < 4) { throw new IOException("malformed category at " + file + " line " + lineNumber); @@ -310,13 +492,16 @@ public final class MecabDictionary { } /** - * Classifies a character. + * Classifies a character by code point, so that a character outside the Basic + * Multilingual Plane is classified as the one character it is rather than as its two + * surrogates. * - * @param c The character. - * @return Its category, falling back to {@code DEFAULT}. Never {@code null}. + * @param codePoint The code point to classify. + * @return Its category, falling back to {@code DEFAULT} when no {@code char.def} + * mapping covers the code point. Never {@code null}. */ - Category categoryOf(char c) { - final String name = categoryOfChar[c]; + Category categoryOf(int codePoint) { + final String name = categoryTable.categoryOf(codePoint); final Category category = name == null ? null : categories.get(name); return category != null ? category : categories.get("DEFAULT"); } @@ -436,15 +621,21 @@ public final class MecabDictionary { * @param text The field text including the {@code 0x} prefix. * @param file The file being read, for the error message. * @param lineNumber The line being read, for the error message. - * @return The parsed code point. - * @throws IOException Thrown if the field is not a valid hexadecimal code point. + * @return The parsed code point, which may be in a supplementary plane. + * @throws IOException Thrown if the field is not a valid hexadecimal code point or + * names a value no Unicode code point has. */ private static int parseCodePoint(String text, Path file, int lineNumber) throws IOException { + final int codePoint; try { - return Integer.parseInt(text.trim().substring(2), 16); + codePoint = Integer.parseInt(text.trim().substring(2), 16); } catch (RuntimeException e) { throw new IOException("malformed code point in " + file + " line " + lineNumber, e); } + if (!Character.isValidCodePoint(codePoint)) { + throw new IOException("code point out of range in " + file + " line " + lineNumber); + } + return codePoint; } } diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeTokenizerTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeTokenizerTest.java index 83b823706..bf7a5f8c3 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeTokenizerTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeTokenizerTest.java @@ -119,6 +119,37 @@ public class LatticeTokenizerTest { Assertions.assertEquals(true, morphemes.get(0).unknown()); } + /** + * Verifies that an unknown-word candidate never spans a character category boundary. + * An unlisted kanji directly followed by a Latin letter must be analyzed as two + * morphemes of their own categories, never as one KANJI morpheme whose surface glues + * the kanji to the letter. + */ + @Test + void testUnknownCandidatesNeverSpanCategoryBoundaries() { + final String text = "\u5CE0a"; + Assertions.assertArrayEquals(new String[] {"\u5CE0", "a"}, tokenizer.tokenize(text)); + Assertions.assertArrayEquals(new Span[] {new Span(0, 1), new Span(1, 2)}, + tokenizer.tokenizePos(text)); + final List<Morpheme> morphemes = tokenizer.analyze(text); + Assertions.assertEquals(List.of("noun", "unknown"), morphemes.get(0).features()); + Assertions.assertEquals(List.of("noun", "foreign"), morphemes.get(1).features()); + } + + /** + * Verifies that bounding unknown-word candidates by the category run does not under + * generate inside the run: a two-kanji unlisted run followed by a Latin letter still + * offers the length-two KANJI candidate, which wins over two single-kanji morphemes. + */ + @Test + void testUnknownRunStillOffersWithinCategoryLengths() { + final String text = "\u5CE0\u9053a"; + Assertions.assertArrayEquals(new String[] {"\u5CE0\u9053", "a"}, + tokenizer.tokenize(text)); + Assertions.assertArrayEquals(new Span[] {new Span(0, 2), new Span(2, 3)}, + tokenizer.tokenizePos(text)); + } + @Test void testWhitespaceSeparatesAndIsNeverAMorpheme() { final String text = "\u6771\u4EAC \u306B \u884C\u304F"; @@ -216,6 +247,8 @@ public class LatticeTokenizerTest { */ @Test void testShortLexiconRowFailsLoud(@TempDir Path broken) throws IOException { + // The rest of the dictionary is well formed, so the short row is what load rejects. + writeUnitMatrixDictionary(broken); Files.write(broken.resolve("lexicon.csv"), "\u6771,0,0\n".getBytes(StandardCharsets.UTF_8)); Assertions.assertThrows(IOException.class, () -> MecabDictionary.load(broken)); @@ -227,6 +260,8 @@ public class LatticeTokenizerTest { */ @Test void testNonNumericLexiconCostFailsLoud(@TempDir Path broken) throws IOException { + // The rest of the dictionary is well formed, so the cost column is what load rejects. + writeUnitMatrixDictionary(broken); Files.write(broken.resolve("lexicon.csv"), "\u6771,0,0,abc,noun\n".getBytes(StandardCharsets.UTF_8)); Assertions.assertThrows(IOException.class, () -> MecabDictionary.load(broken)); @@ -295,6 +330,152 @@ public class LatticeTokenizerTest { Assertions.assertThrows(IOException.class, () -> MecabDictionary.load(broken)); } + /** + * Writes a miniature dictionary whose {@code char.def} maps a supplementary plane + * range, the shape a UniDic-style distribution uses for the CJK extension blocks. + * + * @param target The directory to write the dictionary files into. Must not be + * {@code null} and must exist. + * @throws IOException Thrown if writing any of the files fails. + */ + private static void writeSupplementaryDictionary(Path target) throws IOException { + Files.write(target.resolve("lexicon.csv"), + "\u6771,0,0,6000,noun,common\n".getBytes(StandardCharsets.UTF_8)); + Files.write(target.resolve("matrix.def"), "1 1\n0 0 0\n".getBytes(StandardCharsets.UTF_8)); + Files.write(target.resolve("char.def"), String.join("\n", + "DEFAULT 0 1 0", + "KANJI 0 0 2", + "LATIN 1 1 0", + "", + "0x4E00..0x9FFF KANJI", + "0x20000..0x2A6DF KANJI", + "0x0061..0x007A LATIN", + "").getBytes(StandardCharsets.UTF_8)); + Files.write(target.resolve("unk.def"), String.join("\n", + "DEFAULT,0,0,10000,symbol,unknown", + "KANJI,0,0,8000,noun,unknown", + "LATIN,0,0,4000,noun,foreign", + "").getBytes(StandardCharsets.UTF_8)); + } + + /** + * Verifies that a {@code char.def} range above U+FFFF is honored rather than + * discarded: a supplementary plane ideograph inside the mapped range takes the + * category the range names, while a supplementary code point outside every mapped + * range still falls back to DEFAULT. + */ + @Test + void testSupplementaryCharDefRangeIsHonored(@TempDir Path supplementary) + throws IOException { + writeSupplementaryDictionary(supplementary); + final MecabDictionary dictionary = MecabDictionary.load(supplementary); + // U+20BB7 is a CJK extension B ideograph inside the mapped range. + Assertions.assertEquals("KANJI", dictionary.categoryOf(0x20BB7).name()); + Assertions.assertEquals("KANJI", dictionary.categoryOf(0x6771).name()); + Assertions.assertEquals("DEFAULT", dictionary.categoryOf(0x2460).name()); + Assertions.assertEquals("DEFAULT", dictionary.categoryOf(0x2A6E0).name()); + Assertions.assertEquals("LATIN", dictionary.categoryOf('a').name()); + } + + /** + * Verifies that a supplementary plane ideograph is analyzed as the single character + * it is: one morpheme whose span covers both code units and which carries the + * features of the category its {@code char.def} range names, never one morpheme per + * surrogate. The second case shows the category's length templates count characters, + * not code units, so a run of two supplementary ideographs is still reachable by the + * length-two template. + */ + @Test + void testSupplementaryIdeographIsOneMorpheme(@TempDir Path supplementary) + throws IOException { + writeSupplementaryDictionary(supplementary); + final LatticeTokenizer supplementaryTokenizer = + new LatticeTokenizer(MecabDictionary.load(supplementary)); + // U+20BB7 written as its surrogate pair, per this file's ASCII-only convention. + final String text = "\uD842\uDFB7"; + final List<Morpheme> morphemes = supplementaryTokenizer.analyze(text); + Assertions.assertEquals(1, morphemes.size()); + Assertions.assertEquals(text, morphemes.get(0).surface()); + Assertions.assertEquals(new Span(0, 2), morphemes.get(0).span()); + Assertions.assertEquals(List.of("noun", "unknown"), morphemes.get(0).features()); + + final List<Morpheme> pair = supplementaryTokenizer.analyze(text + text); + Assertions.assertEquals(1, pair.size()); + Assertions.assertEquals(new Span(0, 4), pair.get(0).span()); + Assertions.assertEquals(List.of("noun", "unknown"), pair.get(0).features()); + } + + /** + * Verifies that a supplementary plane ideograph does not absorb neighbouring text of + * another category: the ideograph and an unmapped symbol beside it stay two + * morphemes, each span covering whole characters. + */ + @Test + void testSupplementaryIdeographDoesNotAbsorbItsNeighbour(@TempDir Path supplementary) + throws IOException { + writeSupplementaryDictionary(supplementary); + final LatticeTokenizer supplementaryTokenizer = + new LatticeTokenizer(MecabDictionary.load(supplementary)); + final String text = "\uD842\uDFB7\u2460"; + Assertions.assertArrayEquals(new String[] {"\uD842\uDFB7", "\u2460"}, + supplementaryTokenizer.tokenize(text)); + Assertions.assertArrayEquals(new Span[] {new Span(0, 2), new Span(2, 3)}, + supplementaryTokenizer.tokenizePos(text)); + } + + /** + * Writes every dictionary file except the lexicon, so a test can supply a lexicon of + * its own against a one by one connection matrix. + * + * @param target The directory to write the dictionary files into. Must not be + * {@code null} and must exist. + * @throws IOException Thrown if writing any of the files fails. + */ + private static void writeUnitMatrixDictionary(Path target) throws IOException { + Files.write(target.resolve("matrix.def"), "1 1\n0 0 0\n".getBytes(StandardCharsets.UTF_8)); + Files.write(target.resolve("char.def"), + "DEFAULT 0 1 0\n".getBytes(StandardCharsets.UTF_8)); + Files.write(target.resolve("unk.def"), + "DEFAULT,0,0,10000,symbol,unknown\n".getBytes(StandardCharsets.UTF_8)); + } + + /** + * Verifies that a lexicon row whose right context id is outside the + * {@code matrix.def} dimensions is rejected at load time, naming the file, the line, + * and the offending id, rather than reaching the cost matrix with an out of range + * index during segmentation. + */ + @Test + void testRightContextIdBeyondMatrixFailsLoudAtLoad(@TempDir Path mismatched) + throws IOException { + writeUnitMatrixDictionary(mismatched); + Files.write(mismatched.resolve("lexicon.csv"), + "\u6771,0,5,3000,noun\n".getBytes(StandardCharsets.UTF_8)); + final IOException e = Assertions.assertThrows(IOException.class, + () -> MecabDictionary.load(mismatched)); + Assertions.assertEquals("malformed entry at " + mismatched.resolve("lexicon.csv") + + " line 1: right context id 5 is outside the matrix.def dimensions 1 1", + e.getMessage()); + } + + /** + * Verifies that a lexicon row whose left context id is outside the {@code matrix.def} + * dimensions is rejected at load time, naming the file, the line, and the offending + * id. + */ + @Test + void testLeftContextIdBeyondMatrixFailsLoudAtLoad(@TempDir Path mismatched) + throws IOException { + writeUnitMatrixDictionary(mismatched); + Files.write(mismatched.resolve("lexicon.csv"), + "\u6771,0,0,3000,noun\n\u90FD,7,0,3000,noun\n".getBytes(StandardCharsets.UTF_8)); + final IOException e = Assertions.assertThrows(IOException.class, + () -> MecabDictionary.load(mismatched)); + Assertions.assertEquals("malformed entry at " + mismatched.resolve("lexicon.csv") + + " line 2: left context id 7 is outside the matrix.def dimensions 1 1", + e.getMessage()); + } + @Test void testInvalidArguments() { Assertions.assertThrows(IllegalArgumentException.class,
