This is an automated email from the ASF dual-hosted git repository. krickert pushed a commit to branch OPENNLP-1894-lattice-cjk in repository https://gitbox.apache.org/repos/asf/opennlp.git
commit 8730659ac821636d97444e5a0b449ef8d69c8429 Author: Kristian Rickert <[email protected]> AuthorDate: Thu Jul 16 01:20:06 2026 -0400 OPENNLP-1894: Usage example and edge-case tests for the lattice and unigram segmenters, corrected javadoc --- .../tools/tokenize/lattice/LatticeTokenizer.java | 6 +- .../tools/tokenize/lattice/MecabDictionary.java | 48 ++++- .../tokenize/lattice/MecabDictionaryInstaller.java | 57 +++++- .../tools/tokenize/lattice/UnigramSegmenter.java | 13 +- .../tokenize/lattice/LatticeTokenizerTest.java | 149 ++++++++++++++++ .../tokenize/lattice/LatticeUsageExampleTest.java | 198 +++++++++++++++++++++ .../tokenize/lattice/UnigramSegmenterTest.java | 67 +++++++ 7 files changed, 529 insertions(+), 9 deletions(-) diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/LatticeTokenizer.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/LatticeTokenizer.java index c811fd326..cb610f74f 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/LatticeTokenizer.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/LatticeTokenizer.java @@ -225,7 +225,8 @@ public class LatticeTokenizer implements Tokenizer { } } if (candidates.isEmpty()) { - // no entry and no template: a single-character fallback keeps the lattice alive + // Neither the lexicon nor the character's category produced a candidate here, so a + // single-character entry from the DEFAULT template keeps the lattice connected. final List<WordEntry> fallback = dictionary.unknownEntries("DEFAULT"); if (fallback != null) { for (final WordEntry entry : fallback) { @@ -251,7 +252,8 @@ public class LatticeTokenizer implements Tokenizer { final int lengths = category.length(); for (int length = 1; length <= lengths && position + length <= to; length++) { if (category.group() && position + length == runEnd) { - continue; // already emitted as the grouped run + // This length coincides with the grouped run emitted above; skip the duplicate. + continue; } for (final WordEntry entry : templates) { candidates.add(new Node(position, position + length, entry, true)); diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionary.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionary.java index 2e20b8c2d..908bc75f2 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionary.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionary.java @@ -293,7 +293,7 @@ public final class MecabDictionary { } } - /** @return The longest surface form in the lexicon, bounding prefix enumeration. */ + /** @return The length in characters of the longest surface form in the lexicon. */ int maxSurfaceLength() { return maxSurfaceLength; } @@ -331,6 +331,14 @@ public final class MecabDictionary { return unknownEntries.get(category); } + /** + * Reads a required dictionary file into its lines, accepting LF and CRLF endings. + * + * @param file The file to read. + * @param charset The encoding to decode with. + * @return The lines without their line terminators. Never {@code null}. + * @throws IOException Thrown if the file is missing or reading fails. + */ private static List<String> readLines(Path file, Charset charset) throws IOException { if (!Files.exists(file)) { throw new IOException("required dictionary file is missing: " + file); @@ -351,11 +359,25 @@ public final class MecabDictionary { return lines; } + /** + * Removes a trailing {@code #} comment from a {@code char.def} line. + * + * @param line The raw line. + * @return The line up to but excluding the first {@code #}, or the whole line when + * there is none. + */ private static String stripComment(String line) { final int hash = line.indexOf('#'); return hash < 0 ? line : line.substring(0, hash); } + /** + * Splits a lexicon line at every comma. Surfaces containing commas are not + * representable, matching the plain-text lexicon format. + * + * @param line The line to split. + * @return The fields in order, empty fields included. Never {@code null}. + */ private static List<String> splitCsv(String line) { final List<String> fields = new ArrayList<>(); int start = 0; @@ -368,6 +390,12 @@ public final class MecabDictionary { return fields; } + /** + * Splits a line into its whitespace-separated fields. + * + * @param line The line to split. + * @return The non-empty fields in order. Never {@code null}. + */ private static String[] splitWhitespace(String line) { final List<String> parts = new ArrayList<>(); int start = -1; @@ -384,6 +412,15 @@ public final class MecabDictionary { return parts.toArray(new String[0]); } + /** + * Parses a decimal integer field, reporting the file and line on failure. + * + * @param text The field text. + * @param file The file being read, for the error message. + * @param lineNumber The line being read, for the error message. + * @return The parsed value. + * @throws IOException Thrown if the field is not a valid integer. + */ private static int parseInt(String text, String file, int lineNumber) throws IOException { try { @@ -393,6 +430,15 @@ public final class MecabDictionary { } } + /** + * Parses a {@code 0x}-prefixed hexadecimal code point from {@code char.def}. + * + * @param text The field text including the {@code 0x} prefix. + * @param file The file being read, for the error message. + * @param lineNumber The line being read, for the error message. + * @return The parsed code point. + * @throws IOException Thrown if the field is not a valid hexadecimal code point. + */ private static int parseCodePoint(String text, Path file, int lineNumber) throws IOException { try { diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionaryInstaller.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionaryInstaller.java index 33f3ef58d..9bbbde790 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionaryInstaller.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionaryInstaller.java @@ -34,9 +34,11 @@ import java.util.zip.GZIPInputStream; * built in. * * <p>The installer reads gzip-compressed tar archives, the format the common - * distributions use, and extracts only the files a {@link MecabDictionary} reads: - * {@code *.csv}, {@code *.def}, and {@code dicrc}. Entries are flattened to their base - * names, which also means no archive path can escape the target directory.</p> + * distributions use, and extracts only the dictionary payload: the {@code *.csv} + * lexicon files and {@code *.def} definition files that a {@link MecabDictionary} + * reads, plus the {@code dicrc} configuration file distributions ship alongside them. + * Entries are flattened to their base names, which also means no archive path can + * escape the target directory.</p> * * @since 3.0.0 */ @@ -49,7 +51,7 @@ public final class MecabDictionaryInstaller { private static final int TAR_TYPE_OFFSET = 156; private MecabDictionaryInstaller() { - // static installer only + // This class exposes only static methods and is never instantiated. } /** @@ -122,6 +124,15 @@ public final class MecabDictionaryInstaller { return extracted; } + /** + * Fills one tar block from the stream. + * + * @param in The tar stream. + * @param block The block buffer to fill completely. + * @return {@code true} when a full block was read, {@code false} at a clean end of + * stream before any byte of the block. + * @throws IOException Thrown if the stream ends inside the block or reading fails. + */ private static boolean readBlock(InputStream in, byte[] block) throws IOException { int filled = 0; while (filled < block.length) { @@ -137,6 +148,12 @@ public final class MecabDictionaryInstaller { return true; } + /** + * Recognizes the all-zero block that terminates a tar archive. + * + * @param block The block to inspect. + * @return {@code true} when every byte is zero. + */ private static boolean isEndBlock(byte[] block) { for (final byte b : block) { if (b != 0) { @@ -146,6 +163,12 @@ public final class MecabDictionaryInstaller { return true; } + /** + * Reads the NUL-terminated entry name from a tar header block. + * + * @param header The header block. + * @return The entry name. Never {@code null}. + */ private static String headerName(byte[] header) { int end = 0; while (end < TAR_NAME_LENGTH && header[end] != 0) { @@ -154,6 +177,13 @@ public final class MecabDictionaryInstaller { return new String(header, 0, end, StandardCharsets.UTF_8); } + /** + * Reads the octal entry size from a tar header block. + * + * @param header The header block. + * @return The entry size in bytes. + * @throws IOException Thrown if the size field holds a non-octal digit. + */ private static long headerSize(byte[] header) throws IOException { long size = 0; for (int i = TAR_SIZE_OFFSET; i < TAR_SIZE_OFFSET + TAR_SIZE_LENGTH; i++) { @@ -169,16 +199,35 @@ public final class MecabDictionaryInstaller { return size; } + /** + * Strips any directory prefix from an archive entry name. + * + * @param name The entry name as stored in the archive. + * @return The part after the last {@code /}, or the whole name when there is none. + */ private static String baseName(String name) { final int slash = name.lastIndexOf('/'); return slash < 0 ? name : name.substring(slash + 1); } + /** + * Computes the padding after an entry: tar content is stored in whole blocks. + * + * @param size The entry size in bytes. + * @return The number of padding bytes up to the next block boundary. + */ private static long padding(long size) { final long remainder = size % TAR_BLOCK; return remainder == 0 ? 0 : TAR_BLOCK - remainder; } + /** + * Consumes and discards an exact number of bytes from the stream. + * + * @param in The stream to read from. + * @param bytes The number of bytes to discard. + * @throws IOException Thrown if the stream ends before that many bytes were read. + */ private static void skip(InputStream in, long bytes) throws IOException { long remaining = bytes; final byte[] buffer = new byte[8192]; diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/UnigramSegmenter.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/UnigramSegmenter.java index 3d1ca9884..ea3d188bc 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/UnigramSegmenter.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/UnigramSegmenter.java @@ -167,7 +167,8 @@ public class UnigramSegmenter implements Tokenizer { } node.logProbability = Math.log(entry.getValue()) - logTotal; } - // rarer than any listed word: a fraction of a single count + // Charge an unlisted character half of one count out of the total, which makes it + // rarer than any listed word: every listed count is at least one. final double unknown = Math.log(0.5) - logTotal; return new UnigramSegmenter(root, unknown); } @@ -216,7 +217,8 @@ public class UnigramSegmenter implements Tokenizer { if (best[i] == Double.NEGATIVE_INFINITY) { continue; } - // the single-character fallback keeps every position reachable + // A single-character step at the unknown log-probability keeps every position + // reachable even where no lexicon word matches. final double fallback = best[i] + unknownLogProbability; if (fallback > best[i + 1]) { best[i + 1] = fallback; @@ -247,6 +249,13 @@ public class UnigramSegmenter implements Tokenizer { } } + /** + * Finds the first whitespace character in a lexicon line. + * + * @param text The line to scan. + * @return The index of the first whitespace character, or {@code -1} when the line + * contains none. + */ private static int whitespaceIndex(String text) { for (int i = 0; i < text.length(); i++) { if (StringUtil.isWhitespace(text.charAt(i))) { diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeTokenizerTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeTokenizerTest.java index 1c3f40a8d..b643786a9 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeTokenizerTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeTokenizerTest.java @@ -128,6 +128,155 @@ public class LatticeTokenizerTest { Assertions.assertEquals(0, tokenizer.analyze("").size()); } + /** + * Verifies that empty input yields empty results from every view of the tokenizer. + */ + @Test + void testEmptyInputYieldsEmptyResults() { + Assertions.assertArrayEquals(new String[0], tokenizer.tokenize("")); + Assertions.assertArrayEquals(new Span[0], tokenizer.tokenizePos("")); + } + + /** + * Verifies single-character input for a listed surface and for an unlisted kanji: + * both come back as exactly one morpheme covering {@code [0, 1)}, and only the + * unlisted one is marked unknown. + */ + @Test + void testSingleCharacterInput() { + Assertions.assertArrayEquals(new String[] {"に"}, tokenizer.tokenize("に")); + Assertions.assertArrayEquals(new Span[] {new Span(0, 1)}, tokenizer.tokenizePos("に")); + Assertions.assertFalse(tokenizer.analyze("に").get(0).unknown()); + + final List<Morpheme> unknown = tokenizer.analyze("峠"); + Assertions.assertEquals(1, unknown.size()); + Assertions.assertEquals("峠", unknown.get(0).surface()); + Assertions.assertEquals(new Span(0, 1), unknown.get(0).span()); + Assertions.assertTrue(unknown.get(0).unknown()); + } + + /** + * Verifies input made entirely of characters absent from both the lexicon and the + * {@code char.def} mappings: they fall into the DEFAULT category, whose grouping + * setting joins the whole same-category run into one unknown morpheme carrying the + * DEFAULT template's features. + */ + @Test + void testEntirelyUnknownInputGroupsIntoOneDefaultMorpheme() { + final List<Morpheme> morphemes = tokenizer.analyze("①②③"); + Assertions.assertEquals(1, morphemes.size()); + Assertions.assertEquals("①②③", morphemes.get(0).surface()); + Assertions.assertEquals(new Span(0, 3), morphemes.get(0).span()); + Assertions.assertTrue(morphemes.get(0).unknown()); + Assertions.assertEquals(List.of("symbol", "unknown"), morphemes.get(0).features()); + } + + /** + * Verifies a mixed run of known and unknown text: the lexicon words around an + * unmapped character are kept intact, the unmapped character becomes its own + * unknown morpheme, and every span stays in original text coordinates. + */ + @Test + void testMixedKnownAndUnknownRuns() { + final String text = "東京①に行く"; + Assertions.assertArrayEquals( + new String[] {"東京", "①", "に", "行く"}, + tokenizer.tokenize(text)); + Assertions.assertArrayEquals(new Span[] { + new Span(0, 2), new Span(2, 3), new Span(3, 4), new Span(4, 6)}, + tokenizer.tokenizePos(text)); + final List<Morpheme> morphemes = tokenizer.analyze(text); + Assertions.assertFalse(morphemes.get(0).unknown()); + Assertions.assertTrue(morphemes.get(1).unknown()); + Assertions.assertFalse(morphemes.get(2).unknown()); + } + + /** + * Verifies that spans keep original text coordinates when the interesting content + * does not start at position zero because of leading whitespace. + */ + @Test + void testSpansStayOriginalAfterLeadingWhitespace() { + final String text = " 東京都に行く"; + Assertions.assertArrayEquals( + new String[] {"東京", "都", "に", "行く"}, + tokenizer.tokenize(text)); + Assertions.assertArrayEquals(new Span[] { + new Span(2, 4), new Span(4, 5), new Span(5, 6), new Span(6, 8)}, + tokenizer.tokenizePos(text)); + } + + /** + * Verifies that a lexicon row with fewer than the four mandatory columns is + * rejected at load time. + */ + @Test + void testShortLexiconRowFailsLoud(@TempDir Path broken) throws IOException { + Files.write(broken.resolve("lexicon.csv"), + "東,0,0\n".getBytes(StandardCharsets.UTF_8)); + Assertions.assertThrows(IOException.class, () -> MecabDictionary.load(broken)); + } + + /** + * Verifies that a non-numeric cost column in a lexicon row is rejected at load + * time. + */ + @Test + void testNonNumericLexiconCostFailsLoud(@TempDir Path broken) throws IOException { + Files.write(broken.resolve("lexicon.csv"), + "東,0,0,abc,noun\n".getBytes(StandardCharsets.UTF_8)); + Assertions.assertThrows(IOException.class, () -> MecabDictionary.load(broken)); + } + + /** + * Verifies that a {@code matrix.def} data line with the wrong number of fields is + * rejected at load time. + */ + @Test + void testMalformedMatrixLineFailsLoud(@TempDir Path broken) throws IOException { + Files.write(broken.resolve("lexicon.csv"), + "東,0,0,3000,noun\n".getBytes(StandardCharsets.UTF_8)); + Files.write(broken.resolve("matrix.def"), "1 1\n0 0\n".getBytes(StandardCharsets.UTF_8)); + Assertions.assertThrows(IOException.class, () -> MecabDictionary.load(broken)); + } + + /** + * Verifies that a {@code char.def} code point mapping without a category name is + * rejected at load time. + */ + @Test + void testCharDefMappingWithoutCategoryFailsLoud(@TempDir Path broken) + throws IOException { + Files.write(broken.resolve("lexicon.csv"), + "東,0,0,3000,noun\n".getBytes(StandardCharsets.UTF_8)); + Files.write(broken.resolve("matrix.def"), "1 1\n0 0 0\n".getBytes(StandardCharsets.UTF_8)); + Files.write(broken.resolve("char.def"), + "DEFAULT 0 1 0\n0x4E00..0x9FFF\n".getBytes(StandardCharsets.UTF_8)); + Assertions.assertThrows(IOException.class, () -> MecabDictionary.load(broken)); + } + + /** + * Verifies the fail-loud path when a loadable dictionary cannot cover the input: the + * {@code unk.def} has no DEFAULT template, so a character with neither a lexicon + * entry nor a category template stops segmentation with an exception instead of + * being dropped silently. + */ + @Test + void testMissingDefaultTemplateFailsLoudAtTokenizeTime(@TempDir Path partial) + throws IOException { + Files.write(partial.resolve("lexicon.csv"), + "東,0,0,3000,noun\n".getBytes(StandardCharsets.UTF_8)); + Files.write(partial.resolve("matrix.def"), "1 1\n0 0 0\n".getBytes(StandardCharsets.UTF_8)); + Files.write(partial.resolve("char.def"), + "DEFAULT 0 1 0\nKANJI 0 0 2\n0x4E00..0x9FFF KANJI\n" + .getBytes(StandardCharsets.UTF_8)); + Files.write(partial.resolve("unk.def"), + "KANJI,0,0,8000,noun\n".getBytes(StandardCharsets.UTF_8)); + final LatticeTokenizer limited = + new LatticeTokenizer(MecabDictionary.load(partial)); + Assertions.assertThrows(IllegalStateException.class, () -> limited.analyze("①")); + } + @Test void testMalformedDictionariesFailLoud(@TempDir Path broken) throws IOException { Files.write(broken.resolve("lexicon.csv"), diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeUsageExampleTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeUsageExampleTest.java new file mode 100644 index 000000000..a5e2a39d1 --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeUsageExampleTest.java @@ -0,0 +1,198 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.tokenize.lattice; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.zip.GZIPOutputStream; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import opennlp.tools.util.Span; + +/** + * Demonstrates the intended end-to-end usage of this package with miniature, + * project-authored data: a mecab-format dictionary archive is installed with + * {@link MecabDictionaryInstaller}, loaded as a {@link MecabDictionary}, and segmented + * with a {@link LatticeTokenizer}; a plain frequency lexicon is loaded and segmented + * with a {@link UnigramSegmenter}. Everything is written to a temporary directory by + * the test itself; no external dictionary or lexicon data and no network access are + * involved. + */ +public class LatticeUsageExampleTest { + + /** + * Appends one ustar file entry to a growing tar image: a 512-byte header block + * followed by the content padded to a block boundary. + * + * @param tar The tar image under construction. Must not be {@code null}. + * @param name The entry name including any directory prefix. Must not be + * {@code null}, empty, or longer than the 100-byte header name field. + * @param content The entry content bytes. Must not be {@code null}. + * @throws IOException Thrown if writing to the in-memory stream fails. + * @throws IllegalArgumentException Thrown if {@code name} does not fit the header. + */ + private static void tarEntry(ByteArrayOutputStream tar, String name, byte[] content) + throws IOException { + final byte[] nameBytes = name.getBytes(StandardCharsets.UTF_8); + if (nameBytes.length == 0 || nameBytes.length > 100) { + throw new IllegalArgumentException( + "entry name must be 1..100 bytes, got " + nameBytes.length); + } + final byte[] header = new byte[512]; + System.arraycopy(nameBytes, 0, header, 0, nameBytes.length); + final byte[] mode = "0000644".getBytes(StandardCharsets.US_ASCII); + System.arraycopy(mode, 0, header, 100, mode.length); + final String size = String.format("%011o", content.length); + System.arraycopy(size.getBytes(StandardCharsets.US_ASCII), 0, header, 124, 11); + header[156] = '0'; + for (int i = 148; i < 156; i++) { + header[i] = ' '; + } + int checksum = 0; + for (final byte b : header) { + checksum += b & 0xFF; + } + final String checksumText = String.format("%06o", checksum); + System.arraycopy(checksumText.getBytes(StandardCharsets.US_ASCII), 0, header, 148, 6); + header[154] = 0; + header[155] = ' '; + tar.write(header); + tar.write(content); + final int padding = (512 - content.length % 512) % 512; + tar.write(new byte[padding]); + } + + /** + * Builds a gzip-compressed tar archive from name and content pairs, the layout a + * dictionary distribution ships in. + * + * @param entries The entries as {@code {name, content}} pairs. Must not be + * {@code null}. + * @return The compressed archive bytes. Never {@code null}. + * @throws IOException Thrown if writing to the in-memory streams fails. + */ + private static byte[] gzippedTar(String[][] entries) throws IOException { + final ByteArrayOutputStream tar = new ByteArrayOutputStream(); + for (final String[] entry : entries) { + tarEntry(tar, entry[0], entry[1].getBytes(StandardCharsets.UTF_8)); + } + tar.write(new byte[1024]); + final ByteArrayOutputStream compressed = new ByteArrayOutputStream(); + try (GZIPOutputStream gzip = new GZIPOutputStream(compressed)) { + gzip.write(tar.toByteArray()); + } + return compressed.toByteArray(); + } + + /** + * Walks the full mecab-format flow: package a miniature Japanese dictionary as a + * {@code tar.gz} archive, install it from a file URI, load it, and tokenize. The + * segmentation must pick the cheaper path (Tokyo plus the metropolis suffix) over + * the competing reading (east plus Kyoto), the spans must be in original text + * coordinates, and the morphemes must carry the dictionary's feature columns. + */ + @Test + void testInstallLoadAndTokenizeAMecabFormatDictionary(@TempDir Path work) + throws IOException { + // A minimal but complete dictionary: one lexicon file plus the three definition + // files every mecab-format distribution contains, wrapped like a release archive. + final byte[] archive = gzippedTar(new String[][] { + {"mini-dict-0.1/lexicon.csv", String.join("\n", + "東京,0,0,3000,noun,proper", + "京都,0,0,3000,noun,proper", + "東,0,0,6000,noun,common", + "都,0,0,4000,noun,suffix", + "に,0,0,1000,particle,case", + "行く,0,0,3000,verb,base", + "")}, + {"mini-dict-0.1/matrix.def", "1 1\n0 0 0\n"}, + {"mini-dict-0.1/char.def", String.join("\n", + "DEFAULT 0 1 0", + "KANJI 0 0 2", + "HIRAGANA 0 1 0", + "", + "0x3041..0x3096 HIRAGANA", + "0x4E00..0x9FFF KANJI", + "")}, + {"mini-dict-0.1/unk.def", String.join("\n", + "DEFAULT,0,0,10000,symbol,unknown", + "KANJI,0,0,8000,noun,unknown", + "HIRAGANA,0,0,9000,particle,unknown", + "")}, + {"mini-dict-0.1/README", "not a dictionary payload file"}}); + final Path archiveFile = work.resolve("mini-dict-0.1.tar.gz"); + Files.write(archiveFile, archive); + + // Install: fetch the archive from the user-chosen location and unpack the payload. + final Path dictionaryDirectory = work.resolve("dictionary"); + final int extracted = + MecabDictionaryInstaller.install(archiveFile.toUri(), dictionaryDirectory); + Assertions.assertEquals(4, extracted); + + // Load and tokenize; both views must agree and stay in original coordinates. + final LatticeTokenizer tokenizer = + new LatticeTokenizer(MecabDictionary.load(dictionaryDirectory)); + final String text = "東京都に行く"; + Assertions.assertArrayEquals( + new String[] {"東京", "都", "に", "行く"}, + tokenizer.tokenize(text)); + Assertions.assertArrayEquals(new Span[] { + new Span(0, 2), new Span(2, 3), new Span(3, 4), new Span(4, 6)}, + tokenizer.tokenizePos(text)); + + // The analyze view adds the dictionary's feature columns to every morpheme. + final List<Morpheme> morphemes = tokenizer.analyze(text); + Assertions.assertEquals(4, morphemes.size()); + Assertions.assertEquals("東京", morphemes.get(0).surface()); + Assertions.assertEquals(List.of("noun", "proper"), morphemes.get(0).features()); + Assertions.assertFalse(morphemes.get(0).unknown()); + } + + /** + * Walks the frequency-lexicon flow: write a miniature word-count lexicon to a file, + * load it, and segment. The segmentation must recover the listed multi-character + * words with spans in original text coordinates. + */ + @Test + void testLoadAndSegmentWithAFrequencyLexicon(@TempDir Path work) throws IOException { + // One word, its count, and an optional tag per line, whitespace separated. + final Path lexicon = work.resolve("words.txt"); + Files.write(lexicon, String.join("\n", + "我 5000 r", + "来到 2000 v", + "北京 3000 ns", + "天安门 1200 ns", + "").getBytes(StandardCharsets.UTF_8)); + + final UnigramSegmenter segmenter = UnigramSegmenter.load(lexicon); + final String text = "我来到北京天安门"; + Assertions.assertArrayEquals( + new String[] {"我", "来到", "北京", "天安门"}, + segmenter.tokenize(text)); + Assertions.assertArrayEquals(new Span[] { + new Span(0, 1), new Span(1, 3), new Span(3, 5), new Span(5, 8)}, + segmenter.tokenizePos(text)); + } +} diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/UnigramSegmenterTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/UnigramSegmenterTest.java index a6104c195..f086eafa2 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/UnigramSegmenterTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/UnigramSegmenterTest.java @@ -82,6 +82,73 @@ public class UnigramSegmenterTest { Assertions.assertEquals(0, segmenter.tokenizePos(" ").length); } + /** + * Verifies that empty input yields empty results from both views of the segmenter. + */ + @Test + void testEmptyInputYieldsEmptyResults() { + Assertions.assertArrayEquals(new String[0], segmenter.tokenize("")); + Assertions.assertArrayEquals(new Span[0], segmenter.tokenizePos("")); + } + + /** + * Verifies single-character input for a listed word and for a character the + * lexicon does not know: both come back as exactly one token covering + * {@code [0, 1)}. + */ + @Test + void testSingleCharacterInput() { + Assertions.assertArrayEquals(new String[] {"我"}, segmenter.tokenize("我")); + Assertions.assertArrayEquals(new Span[] {new Span(0, 1)}, segmenter.tokenizePos("我")); + Assertions.assertArrayEquals(new String[] {"爱"}, segmenter.tokenize("爱")); + Assertions.assertArrayEquals(new Span[] {new Span(0, 1)}, segmenter.tokenizePos("爱")); + } + + /** + * Verifies input made entirely of characters absent from the lexicon: every + * character becomes its own single-character token, since only the unknown + * fallback is available. + */ + @Test + void testEntirelyUnknownInputFallsBackToSingleCharacters() { + Assertions.assertArrayEquals( + new String[] {"x", "y", "z"}, + segmenter.tokenize("xyz")); + Assertions.assertArrayEquals(new Span[] { + new Span(0, 1), new Span(1, 2), new Span(2, 3)}, + segmenter.tokenizePos("xyz")); + } + + /** + * Verifies a mixed run of known and unknown text inside one whitespace-free + * stretch: the unknown character becomes a single token while the listed words + * around it, including the longest listed compound, stay intact. + */ + @Test + void testMixedKnownAndUnknownRuns() { + Assertions.assertArrayEquals( + new String[] {"我", "爱", "清华大学"}, + segmenter.tokenize("我爱清华大学")); + Assertions.assertArrayEquals(new Span[] { + new Span(0, 1), new Span(1, 2), new Span(2, 6)}, + segmenter.tokenizePos("我爱清华大学")); + } + + /** + * Verifies that spans keep original text coordinates when the content does not + * start at position zero because of leading whitespace. + */ + @Test + void testSpansStayOriginalAfterLeadingWhitespace() { + final String text = " 我来到北京"; + Assertions.assertArrayEquals( + new String[] {"我", "来到", "北京"}, + segmenter.tokenize(text)); + Assertions.assertArrayEquals(new Span[] { + new Span(2, 3), new Span(3, 5), new Span(5, 7)}, + segmenter.tokenizePos(text)); + } + @Test void testMalformedLexiconsFailLoud() { Assertions.assertThrows(IOException.class, () -> UnigramSegmenter.load(
