This is an automated email from the ASF dual-hosted git repository. krickert pushed a commit to branch OPENNLP-1894-lattice-cjk in repository https://gitbox.apache.org/repos/asf/opennlp.git
commit b5723b45d79936c0e409a49a848f068a660567e7 Author: Kristian Rickert <[email protected]> AuthorDate: Tue Jul 21 06:39:29 2026 -0400 OPENNLP-1894: Apply the review-convention pass and drop unreferenced lexicon accessors --- .../dev => dev}/README-mecab-dictionaries.md | 10 ++- .../dev => dev}/download-mecab-dictionary.sh | 4 +- .../tools/tokenize/lattice/LatticeTokenizer.java | 13 +-- .../tools/tokenize/lattice/MecabDictionary.java | 93 +++++---------------- .../tokenize/lattice/MecabDictionaryInstaller.java | 22 +++-- .../tools/tokenize/lattice/UnigramSegmenter.java | 19 +++-- .../tokenize/lattice/LatticeTokenizerTest.java | 6 +- .../tokenize/lattice/LatticeUsageExampleTest.java | 68 +-------------- .../lattice/MecabDictionaryInstallerTest.java | 55 ++---------- .../tools/tokenize/lattice/TarGzArchives.java | 97 ++++++++++++++++++++++ .../tokenize/lattice/UnigramSegmenterTest.java | 8 +- opennlp-docs/src/docbkx/tokenizer.xml | 2 +- 12 files changed, 171 insertions(+), 226 deletions(-) diff --git a/opennlp-core/opennlp-runtime/dev/README-mecab-dictionaries.md b/dev/README-mecab-dictionaries.md similarity index 88% rename from opennlp-core/opennlp-runtime/dev/README-mecab-dictionaries.md rename to dev/README-mecab-dictionaries.md index c3244ba9c..fce653b39 100644 --- a/opennlp-core/opennlp-runtime/dev/README-mecab-dictionaries.md +++ b/dev/README-mecab-dictionaries.md @@ -17,7 +17,7 @@ # CJK dictionaries for the lattice tokenizer -The lattice tokenizer (`opennlp.tools.tokenize.lattice`) segments Japanese and Korean over a mecab-format dictionary, and the unigram segmenter handles Chinese over a plain word-frequency lexicon. Apache OpenNLP bundles no dictionary data: you download a dictionary from the project of your choice, and by using it you accept that dictionary's license. Read the license file inside the archive before use; nothing in this repository grants you those terms. +The lattice tokenizer (`opennlp.tools.tokenize.lattice`) segments Japanese and Korean over a mecab-format dictionary, and the unigram segmenter handles Chinese over a plain word-frequency lexicon. Apache OpenNLP bundles no dictionary data: you download a dictionary from the project of your choice, and each dictionary carries its own license. Read the license file inside the archive before use. ## Known mecab-format dictionary projects @@ -63,7 +63,8 @@ import opennlp.tools.tokenize.lattice.MecabDictionary; MecabDictionary dictionary = MecabDictionary.load(Path.of("ipadic"), Charset.forName("EUC-JP")); LatticeTokenizer tokenizer = new LatticeTokenizer(dictionary); -String[] tokens = tokenizer.tokenize("東京都に行く"); +// "Tokyo-to ni iku" (go to the Tokyo metropolis), escaped to keep this file ASCII +String[] tokens = tokenizer.tokenize("\u6771\u4EAC\u90FD\u306B\u884C\u304F"); ``` For a UTF-8 dictionary such as mecab-ko-dic, `MecabDictionary.load(Path.of("ko-dic"))` is enough. Loaded dictionaries and tokenizers are immutable and safe to share between threads, so load once and reuse. @@ -77,7 +78,8 @@ import java.nio.file.Path; import opennlp.tools.tokenize.lattice.UnigramSegmenter; UnigramSegmenter segmenter = UnigramSegmenter.load(Path.of("words.txt")); -String[] tokens = segmenter.tokenize("我来到北京天安门"); +// "wo laidao Beijing Tian'anmen" (I arrive at Beijing Tiananmen), escaped as above +String[] tokens = segmenter.tokenize("\u6211\u6765\u5230\u5317\u4EAC\u5929\u5B89\u95E8"); ``` -As with the dictionaries, the lexicon's license is between you and its publisher; nothing is bundled. +As with the dictionaries, the lexicon carries its own license; nothing is bundled. diff --git a/opennlp-core/opennlp-runtime/dev/download-mecab-dictionary.sh b/dev/download-mecab-dictionary.sh similarity index 94% rename from opennlp-core/opennlp-runtime/dev/download-mecab-dictionary.sh rename to dev/download-mecab-dictionary.sh index 30c5b23bd..3b86ed09d 100755 --- a/opennlp-core/opennlp-runtime/dev/download-mecab-dictionary.sh +++ b/dev/download-mecab-dictionary.sh @@ -23,8 +23,8 @@ # this directory for the known dictionary projects, their encodings, and the Java # steps that follow the download. # -# The dictionary you download carries its own license, which you accept by using it. -# Nothing is bundled with Apache OpenNLP and no dictionary location is built in. +# The dictionary you download carries its own license. Nothing is bundled with +# Apache OpenNLP and no dictionary location is built in. set -euo pipefail diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/LatticeTokenizer.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/LatticeTokenizer.java index 98f75d1b6..5ee2384ba 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/LatticeTokenizer.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/LatticeTokenizer.java @@ -69,8 +69,7 @@ public class LatticeTokenizer implements Tokenizer { /** * One lattice node: a candidate morpheme with its best path cost so far. Nodes - * ending at one position chain through {@link #nextEndingHere}, so the lattice - * needs one head reference per position instead of a list allocation. + * ending at one position chain through {@link #nextEndingHere}. */ private static final class Node { private final int start; @@ -141,13 +140,9 @@ public class LatticeTokenizer implements Tokenizer { /** Runs the Viterbi search over one whitespace-free stretch of text. */ private void decode(String text, int from, int to, List<Morpheme> morphemes) { final int length = to - from; - // One head reference per position; nodes ending there chain through the node's - // own link, so building the lattice allocates nothing besides the nodes. + // Each element heads the chain of nodes ending at that position. final Node[] endingAt = new Node[length + 1]; - // One right-to-left pass fixes each position's category and same-category run end, - // so candidate generation reads them instead of rescanning the run from every - // position, which would cost the square of the run length on long uniform runs. final Category[] categoryAt = new Category[length]; final int[] runEndAt = new int[length]; computeCategoryRuns(text, from, to, categoryAt, runEndAt); @@ -259,9 +254,9 @@ public class LatticeTokenizer implements Tokenizer { final Category category; final int runEnd; if (positionCategory == null) { - // Only a lexicon surface ending inside a surrogate pair could make such a + // Only a lexicon surface ending inside a surrogate pair can make such a // position reachable; classify the stray code unit on the spot so the lattice - // stays connected the way it always did. + // stays connected. category = dictionary.categoryOf(codePoint); runEnd = position + Character.charCount(codePoint); } else { diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionary.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionary.java index cd1f46ba7..ed0775a19 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionary.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionary.java @@ -35,9 +35,8 @@ import opennlp.tools.util.StringUtil; * An immutable, in-memory dictionary in the mecab directory format: lexicon entries * from the {@code *.csv} files, connection costs from {@code matrix.def}, character * categories from {@code char.def}, and unknown-word templates from {@code unk.def}, - * loaded from a user-supplied dictionary directory. The reader is a clean-room - * implementation of the documented format; no dictionary data is bundled or downloaded - * by this class, so the dictionaries' own licenses never attach to this library. + * loaded from a user-supplied dictionary directory. No dictionary data is bundled or + * downloaded by this class. * * <p>The same format serves multiple languages: the Japanese IPADIC and UniDic * distributions and the Korean mecab-ko-dic all load through this one reader, with the @@ -60,12 +59,9 @@ public final class MecabDictionary { /** * The lexicon as a double-array trie: one transition is one array read and one - * comparison, so the per-position prefix walk of the tokenizer never hashes, never - * binary-searches a fan-out, and never boxes. Characters are recoded into dense - * labels ordered by frequency before the array is built, which keeps the array - * compact although CJK surfaces draw from tens of thousands of distinct - * characters; a character the lexicon never uses misses in the recode table before - * the array is even consulted. + * comparison. Characters are recoded into dense labels ordered by descending + * frequency before the array is built, which keeps the array compact; a character + * the lexicon never uses misses in the recode table before the array is consulted. * * <p>The layout is the classic base/check pair: from state {@code s}, label * {@code c} leads to {@code t = base[s] + c} exactly when {@code check[t] == s}. @@ -160,32 +156,6 @@ public final class MecabDictionary { } } - /** - * Looks up the entries of an exact surface form. - * - * @param surface The surface form. - * @return The entries, or {@code null} when the surface is not listed. - */ - private List<WordEntry> lookup(String surface) { - int state = Builder.ROOT; - for (int i = 0; i < surface.length(); i++) { - final int code = codeOf[surface.charAt(i)]; - if (code < 0) { - return null; - } - final int next = base[state] + code; - if (next >= check.length || check[next] != state) { - return null; - } - state = next; - } - final int terminal = base[state]; - if (terminal < check.length && check[terminal] == state && base[terminal] < 0) { - return values[-base[terminal] - 1]; - } - return null; - } - /** * The recursive sorted-range builder: each call places one node's children by * finding a base at which every child label lands on a free slot, then recurses @@ -310,12 +280,9 @@ public final class MecabDictionary { * The {@code char.def} code point to category name mapping, over the whole Unicode * code point range. * - * <p>The Basic Multilingual Plane is held in a directly indexed array, which is one - * reference per BMP code point and is the entire mapping for a BMP-only dictionary. - * The supplementary planes are held as a sorted, non-overlapping range table searched - * by binary search, because dictionaries map them in a handful of large blocks: a - * directly indexed array over all of Unicode would spend more than four megabytes of - * references to say what a few range rows already say.</p> + * <p>The Basic Multilingual Plane is held in a directly indexed array. The + * supplementary planes are held as a sorted, non-overlapping range table searched by + * binary search, because dictionaries map them in a handful of large blocks.</p> */ private static final class CategoryTable { @@ -334,10 +301,8 @@ public final class MecabDictionary { /** * Looks up the category a {@code char.def} mapping gives a code point. The table - * holds the {@link Category} instances themselves, so a lookup on the tokenizer's - * per-character path costs one array read or one binary search, never a name map - * access, and two code points of one category share one instance to compare by - * identity. + * holds the {@link Category} instances themselves, and two code points of one + * category share one instance, so categories may be compared by identity. * * @param codePoint The code point to classify. * @return The category, or {@code null} when no mapping covers the code point. @@ -488,7 +453,6 @@ public final class MecabDictionary { } private final DoubleArrayLexicon lexicon; - private final int maxSurfaceLength; private final short[] connectionCosts; private final int rightSize; private final Map<String, Category> categories; @@ -496,11 +460,10 @@ public final class MecabDictionary { private final Category defaultCategory; private final Map<String, List<WordEntry>> unknownEntries; - private MecabDictionary(DoubleArrayLexicon lexicon, int maxSurfaceLength, + private MecabDictionary(DoubleArrayLexicon lexicon, short[] connectionCosts, int rightSize, Map<String, Category> categories, CategoryTable categoryTable, Map<String, List<WordEntry>> unknownEntries) { this.lexicon = lexicon; - this.maxSurfaceLength = maxSurfaceLength; this.connectionCosts = connectionCosts; this.rightSize = rightSize; this.categories = categories; @@ -536,8 +499,11 @@ public final class MecabDictionary { * @throws IllegalArgumentException Thrown if a parameter is {@code null}. */ public static MecabDictionary load(Path directory, Charset charset) throws IOException { - if (directory == null || charset == null) { - throw new IllegalArgumentException("directory and charset must not be null"); + if (directory == null) { + throw new IllegalArgumentException("directory must not be null"); + } + if (charset == null) { + throw new IllegalArgumentException("charset must not be null"); } // The connection matrix is read first because its dimensions are what every // lexicon entry's context ids have to be inside of. @@ -587,11 +553,9 @@ public final class MecabDictionary { } final Map<String, List<WordEntry>> lexicon = new HashMap<>(); - int maxSurface = 0; try (DirectoryStream<Path> csvFiles = Files.newDirectoryStream(directory, "*.csv")) { for (final Path csv : csvFiles) { - maxSurface = Math.max(maxSurface, - readLexicon(csv, charset, lexicon, leftSize, rightSize)); + readLexicon(csv, charset, lexicon, leftSize, rightSize); } } if (lexicon.isEmpty()) { @@ -605,7 +569,7 @@ public final class MecabDictionary { final Map<String, List<WordEntry>> unknown = new HashMap<>(); readLexicon(directory.resolve("unk.def"), charset, unknown, leftSize, rightSize); - return new MecabDictionary(DoubleArrayLexicon.build(lexicon), maxSurface, costs, + return new MecabDictionary(DoubleArrayLexicon.build(lexicon), costs, rightSize, categories, categoryTable.build(categories), unknown); } @@ -620,14 +584,12 @@ public final class MecabDictionary { * ids. * @param rightSize The second {@code matrix.def} dimension, which bounds left context * ids. - * @return The length in characters of the longest surface form read. * @throws IOException Thrown if the file is missing, an entry is malformed, or an * entry's context id is outside the matrix dimensions. */ - private static int readLexicon(Path file, Charset charset, + private static void readLexicon(Path file, Charset charset, Map<String, List<WordEntry>> target, int leftSize, int rightSize) throws IOException { - int maxSurface = 0; int lineNumber = 0; for (final String line : readLines(file, charset)) { lineNumber++; @@ -658,9 +620,7 @@ public final class MecabDictionary { parseInt(fields.get(3), file.toString(), lineNumber), List.copyOf(fields.subList(4, fields.size()))); target.computeIfAbsent(surface, key -> new ArrayList<>(1)).add(entry); - maxSurface = Math.max(maxSurface, surface.length()); } - return maxSurface; } /** Reads char.def: category behavior lines and code point mapping lines. */ @@ -708,16 +668,6 @@ public final class MecabDictionary { } } - /** - * Looks up the lexicon entries for an exact surface form. - * - * @param surface The surface form. - * @return The entries, or {@code null} when the surface is not listed. - */ - List<WordEntry> lookup(String surface) { - return lexicon.lookup(surface); - } - /** * Reports every lexicon surface starting at a text position, walking the trie once * with no substring allocation. @@ -731,11 +681,6 @@ public final class MecabDictionary { lexicon.prefixMatches(text, from, to, consumer); } - /** @return The length in characters of the longest surface form in the lexicon. */ - int maxSurfaceLength() { - return maxSurfaceLength; - } - /** * Reads the connection cost between two adjacent nodes. * diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionaryInstaller.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionaryInstaller.java index 9bbbde790..3c2589bc0 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionaryInstaller.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionaryInstaller.java @@ -29,9 +29,8 @@ import java.util.zip.GZIPInputStream; /** * Fetches and unpacks a mecab-format dictionary archive into a local directory, so the * dictionary is acquired by the user at install time and never ships with this library. - * The user supplies the archive location from the dictionary project of their choice - * and thereby accepts that dictionary's license; nothing is bundled and no location is - * built in. + * The archive location is user-supplied; no dictionary data is bundled and no download + * location is built in. * * <p>The installer reads gzip-compressed tar archives, the format the common * distributions use, and extracts only the dictionary payload: the {@code *.csv} @@ -64,11 +63,15 @@ public final class MecabDictionaryInstaller { * @return The number of dictionary files extracted. * @throws IOException Thrown if fetching, reading, or writing fails, or the archive * contains no dictionary file. - * @throws IllegalArgumentException Thrown if a parameter is {@code null}. + * @throws IllegalArgumentException Thrown if a parameter is {@code null} or + * {@code archive} is not an absolute URI. */ public static int install(URI archive, Path targetDirectory) throws IOException { - if (archive == null || targetDirectory == null) { - throw new IllegalArgumentException("archive and targetDirectory must not be null"); + if (archive == null) { + throw new IllegalArgumentException("archive must not be null"); + } + if (targetDirectory == null) { + throw new IllegalArgumentException("targetDirectory must not be null"); } try (InputStream in = archive.toURL().openStream()) { return extract(in, targetDirectory); @@ -89,8 +92,11 @@ public final class MecabDictionaryInstaller { */ public static int extract(InputStream archiveStream, Path targetDirectory) throws IOException { - if (archiveStream == null || targetDirectory == null) { - throw new IllegalArgumentException("stream and targetDirectory must not be null"); + if (archiveStream == null) { + throw new IllegalArgumentException("archiveStream must not be null"); + } + if (targetDirectory == null) { + throw new IllegalArgumentException("targetDirectory must not be null"); } Files.createDirectories(targetDirectory); final InputStream tar = new GZIPInputStream(archiveStream); diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/UnigramSegmenter.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/UnigramSegmenter.java index b4cfaa8ea..ba3e06861 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/UnigramSegmenter.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/UnigramSegmenter.java @@ -41,9 +41,8 @@ import opennlp.tools.util.StringUtil; * and counts. * * <p>The lexicon format is one entry per line: the word, its count, and optionally a - * tag, separated by whitespace. The user supplies the lexicon file and thereby accepts - * its license; nothing is bundled. Every reported span is in original text - * coordinates.</p> + * tag, separated by whitespace. The lexicon file is user-supplied; no lexicon data is + * bundled. Every reported span is in original text coordinates.</p> * * <p>Instances are immutable and safe to share between threads.</p> * @@ -90,8 +89,11 @@ public class UnigramSegmenter implements Tokenizer { * @throws IllegalArgumentException Thrown if a parameter is {@code null}. */ public static UnigramSegmenter load(Path lexicon, Charset charset) throws IOException { - if (lexicon == null || charset == null) { - throw new IllegalArgumentException("lexicon and charset must not be null"); + if (lexicon == null) { + throw new IllegalArgumentException("lexicon must not be null"); + } + if (charset == null) { + throw new IllegalArgumentException("charset must not be null"); } try (InputStream in = Files.newInputStream(lexicon)) { return load(in, charset); @@ -109,8 +111,11 @@ public class UnigramSegmenter implements Tokenizer { */ public static UnigramSegmenter load(InputStream lexiconStream, Charset charset) throws IOException { - if (lexiconStream == null || charset == null) { - throw new IllegalArgumentException("stream and charset must not be null"); + if (lexiconStream == null) { + throw new IllegalArgumentException("lexiconStream must not be null"); + } + if (charset == null) { + throw new IllegalArgumentException("charset must not be null"); } final Map<String, Long> counts = new HashMap<>(); long total = 0; diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeTokenizerTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeTokenizerTest.java index e1be5a4a9..034cd6d8c 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeTokenizerTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeTokenizerTest.java @@ -99,7 +99,7 @@ public class LatticeTokenizerTest { Assertions.assertEquals(4, morphemes.size()); Assertions.assertEquals(List.of("noun", "proper"), morphemes.get(0).features()); Assertions.assertEquals(List.of("particle", "case"), morphemes.get(2).features()); - Assertions.assertEquals(false, morphemes.get(0).unknown()); + Assertions.assertFalse(morphemes.get(0).unknown()); } @Test @@ -107,7 +107,7 @@ public class LatticeTokenizerTest { final List<Morpheme> morphemes = tokenizer.analyze("ABC\u306B\u884C\u304F"); Assertions.assertEquals(3, morphemes.size()); Assertions.assertEquals("ABC", morphemes.get(0).surface()); - Assertions.assertEquals(true, morphemes.get(0).unknown()); + Assertions.assertTrue(morphemes.get(0).unknown()); Assertions.assertEquals(List.of("noun", "foreign"), morphemes.get(0).features()); } @@ -116,7 +116,7 @@ public class LatticeTokenizerTest { final List<Morpheme> morphemes = tokenizer.analyze("\u5CE0\u9053\u306B\u884C\u304F"); Assertions.assertEquals(3, morphemes.size()); Assertions.assertEquals("\u5CE0\u9053", morphemes.get(0).surface()); - Assertions.assertEquals(true, morphemes.get(0).unknown()); + Assertions.assertTrue(morphemes.get(0).unknown()); } /** diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeUsageExampleTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeUsageExampleTest.java index a61969c90..0fc58585d 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeUsageExampleTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeUsageExampleTest.java @@ -17,14 +17,12 @@ package opennlp.tools.tokenize.lattice; -import java.io.ByteArrayOutputStream; import java.io.IOException; import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.util.List; -import java.util.zip.GZIPOutputStream; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -50,70 +48,6 @@ import opennlp.tools.util.Span; */ public class LatticeUsageExampleTest { - /** - * Appends one ustar file entry to a growing tar image: a 512-byte header block - * followed by the content padded to a block boundary. - * - * @param tar The tar image under construction. Must not be {@code null}. - * @param name The entry name including any directory prefix. Must not be - * {@code null}, empty, or longer than the 100-byte header name field. - * @param content The entry content bytes. Must not be {@code null}. - * @throws IOException Thrown if writing to the in-memory stream fails. - * @throws IllegalArgumentException Thrown if {@code name} does not fit the header. - */ - private static void tarEntry(ByteArrayOutputStream tar, String name, byte[] content) - throws IOException { - final byte[] nameBytes = name.getBytes(StandardCharsets.UTF_8); - if (nameBytes.length == 0 || nameBytes.length > 100) { - throw new IllegalArgumentException( - "entry name must be 1..100 bytes, got " + nameBytes.length); - } - final byte[] header = new byte[512]; - System.arraycopy(nameBytes, 0, header, 0, nameBytes.length); - final byte[] mode = "0000644".getBytes(StandardCharsets.US_ASCII); - System.arraycopy(mode, 0, header, 100, mode.length); - final String size = String.format("%011o", content.length); - System.arraycopy(size.getBytes(StandardCharsets.US_ASCII), 0, header, 124, 11); - header[156] = '0'; - for (int i = 148; i < 156; i++) { - header[i] = ' '; - } - int checksum = 0; - for (final byte b : header) { - checksum += b & 0xFF; - } - final String checksumText = String.format("%06o", checksum); - System.arraycopy(checksumText.getBytes(StandardCharsets.US_ASCII), 0, header, 148, 6); - header[154] = 0; - header[155] = ' '; - tar.write(header); - tar.write(content); - final int padding = (512 - content.length % 512) % 512; - tar.write(new byte[padding]); - } - - /** - * Builds a gzip-compressed tar archive from name and content pairs, the layout a - * dictionary distribution ships in. - * - * @param entries The entries as {@code {name, content}} pairs. Must not be - * {@code null}. - * @return The compressed archive bytes. Never {@code null}. - * @throws IOException Thrown if writing to the in-memory streams fails. - */ - private static byte[] gzippedTar(String[][] entries) throws IOException { - final ByteArrayOutputStream tar = new ByteArrayOutputStream(); - for (final String[] entry : entries) { - tarEntry(tar, entry[0], entry[1].getBytes(StandardCharsets.UTF_8)); - } - tar.write(new byte[1024]); - final ByteArrayOutputStream compressed = new ByteArrayOutputStream(); - try (GZIPOutputStream gzip = new GZIPOutputStream(compressed)) { - gzip.write(tar.toByteArray()); - } - return compressed.toByteArray(); - } - /** * Walks the full mecab-format flow: package a miniature Japanese dictionary as a * {@code tar.gz} archive, install it from a file URI, load it, and tokenize. The @@ -126,7 +60,7 @@ public class LatticeUsageExampleTest { throws IOException { // A minimal but complete dictionary: one lexicon file plus the three definition // files every mecab-format distribution contains, wrapped like a release archive. - final byte[] archive = gzippedTar(new String[][] { + final byte[] archive = TarGzArchives.gzippedTar(new String[][] { {"mini-dict-0.1/lexicon.csv", String.join("\n", "\u6771\u4EAC,0,0,3000,noun,proper", "\u4EAC\u90FD,0,0,3000,noun,proper", diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/MecabDictionaryInstallerTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/MecabDictionaryInstallerTest.java index fe999e460..da95c9736 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/MecabDictionaryInstallerTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/MecabDictionaryInstallerTest.java @@ -18,64 +18,24 @@ package opennlp.tools.tokenize.lattice; import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; -import java.util.zip.GZIPOutputStream; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; +/** + * Tests the installer against project-authored, in-memory archives; no external + * dictionary data and no network access are involved. + */ public class MecabDictionaryInstallerTest { - /** Writes one ustar entry: a 512-byte header block and padded content. */ - private static void tarEntry(ByteArrayOutputStream tar, String name, byte[] content) - throws IOException { - final byte[] header = new byte[512]; - final byte[] nameBytes = name.getBytes(StandardCharsets.UTF_8); - System.arraycopy(nameBytes, 0, header, 0, nameBytes.length); - final byte[] mode = "0000644".getBytes(StandardCharsets.US_ASCII); - System.arraycopy(mode, 0, header, 100, mode.length); - final String size = String.format("%011o", content.length); - System.arraycopy(size.getBytes(StandardCharsets.US_ASCII), 0, header, 124, 11); - header[156] = '0'; - for (int i = 148; i < 156; i++) { - header[i] = ' '; - } - int checksum = 0; - for (final byte b : header) { - checksum += b & 0xFF; - } - final String checksumText = String.format("%06o", checksum); - System.arraycopy(checksumText.getBytes(StandardCharsets.US_ASCII), 0, header, 148, 6); - header[154] = 0; - header[155] = ' '; - tar.write(header); - tar.write(content); - final int padding = (512 - content.length % 512) % 512; - tar.write(new byte[padding]); - } - - private static byte[] archive(String[][] entries) throws IOException { - final ByteArrayOutputStream tar = new ByteArrayOutputStream(); - for (final String[] entry : entries) { - tarEntry(tar, entry[0], entry[1].getBytes(StandardCharsets.UTF_8)); - } - tar.write(new byte[1024]); - final ByteArrayOutputStream compressed = new ByteArrayOutputStream(); - try (GZIPOutputStream gzip = new GZIPOutputStream(compressed)) { - gzip.write(tar.toByteArray()); - } - return compressed.toByteArray(); - } - @Test void testExtractsDictionaryFilesAndFlattensPaths(@TempDir Path target) throws IOException { - final byte[] archive = archive(new String[][] { + final byte[] archive = TarGzArchives.gzippedTar(new String[][] { {"dict-1.0/lexicon.csv", "cat,0,0,100,noun\n"}, {"dict-1.0/matrix.def", "1 1\n0 0 0\n"}, {"dict-1.0/char.def", "DEFAULT 0 1 0\n"}, @@ -101,7 +61,7 @@ public class MecabDictionaryInstallerTest { void testInstallReadsAFileUri(@TempDir Path source, @TempDir Path target) throws IOException { final Path archiveFile = source.resolve("dict.tar.gz"); - Files.write(archiveFile, archive(new String[][] { + Files.write(archiveFile, TarGzArchives.gzippedTar(new String[][] { {"d/words.csv", "cat,0,0,100,noun\n"}, {"d/matrix.def", "1 1\n0 0 0\n"}})); @@ -115,7 +75,8 @@ public class MecabDictionaryInstallerTest { @Test void testArchivesWithoutDictionaryFilesFailLoud(@TempDir Path target) throws IOException { - final byte[] archive = archive(new String[][] {{"readme.txt", "nothing here"}}); + final byte[] archive = + TarGzArchives.gzippedTar(new String[][] {{"readme.txt", "nothing here"}}); Assertions.assertThrows(IOException.class, () -> MecabDictionaryInstaller.extract( new ByteArrayInputStream(archive), target)); } diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/TarGzArchives.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/TarGzArchives.java new file mode 100644 index 000000000..4fdfb29d0 --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/TarGzArchives.java @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.tokenize.lattice; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.zip.GZIPOutputStream; + +/** + * Builds miniature, project-authored gzip-compressed ustar archives in memory for the + * tests of this package; no external archive data is involved. + */ +final class TarGzArchives { + + private TarGzArchives() { + } + + /** + * Builds a gzip-compressed tar archive from name and content pairs, the layout a + * dictionary distribution ships in. + * + * @param entries The entries as {@code {name, content}} pairs. Must not be + * {@code null}. + * @return The compressed archive bytes. Never {@code null}. + * @throws IOException Thrown if writing to the in-memory streams fails. + */ + static byte[] gzippedTar(String[][] entries) throws IOException { + final ByteArrayOutputStream tar = new ByteArrayOutputStream(); + for (final String[] entry : entries) { + tarEntry(tar, entry[0], entry[1].getBytes(StandardCharsets.UTF_8)); + } + tar.write(new byte[1024]); + final ByteArrayOutputStream compressed = new ByteArrayOutputStream(); + try (GZIPOutputStream gzip = new GZIPOutputStream(compressed)) { + gzip.write(tar.toByteArray()); + } + return compressed.toByteArray(); + } + + /** + * Appends one ustar file entry to a growing tar image: a 512-byte header block + * followed by the content padded to a block boundary. + * + * @param tar The tar image under construction. Must not be {@code null}. + * @param name The entry name including any directory prefix. Must not be + * {@code null}, empty, or longer than the 100-byte header name field. + * @param content The entry content bytes. Must not be {@code null}. + * @throws IOException Thrown if writing to the in-memory stream fails. + * @throws IllegalArgumentException Thrown if {@code name} does not fit the header. + */ + private static void tarEntry(ByteArrayOutputStream tar, String name, byte[] content) + throws IOException { + final byte[] nameBytes = name.getBytes(StandardCharsets.UTF_8); + if (nameBytes.length == 0 || nameBytes.length > 100) { + throw new IllegalArgumentException( + "entry name must be 1..100 bytes, got " + nameBytes.length); + } + final byte[] header = new byte[512]; + System.arraycopy(nameBytes, 0, header, 0, nameBytes.length); + final byte[] mode = "0000644".getBytes(StandardCharsets.US_ASCII); + System.arraycopy(mode, 0, header, 100, mode.length); + final String size = String.format("%011o", content.length); + System.arraycopy(size.getBytes(StandardCharsets.US_ASCII), 0, header, 124, 11); + header[156] = '0'; + for (int i = 148; i < 156; i++) { + header[i] = ' '; + } + int checksum = 0; + for (final byte b : header) { + checksum += b & 0xFF; + } + final String checksumText = String.format("%06o", checksum); + System.arraycopy(checksumText.getBytes(StandardCharsets.US_ASCII), 0, header, 148, 6); + header[154] = 0; + header[155] = ' '; + tar.write(header); + tar.write(content); + final int padding = (512 - content.length % 512) % 512; + tar.write(new byte[padding]); + } +} diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/UnigramSegmenterTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/UnigramSegmenterTest.java index e6ebd604e..f5cff587b 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/UnigramSegmenterTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/UnigramSegmenterTest.java @@ -183,19 +183,19 @@ public class UnigramSegmenterTest { * span over both of its surrogate halves, and no span boundary ever lands between * them. */ - @org.junit.jupiter.api.Test + @Test void testUnknownSupplementaryCharacterIsNeverSplit() { // U+20BB7, a CJK extension B ideograph, written as its surrogate pair final String text = "\uD842\uDFB7\uD842\uDFB7"; - final opennlp.tools.util.Span[] spans = segmenter.tokenizePos(text); - for (final opennlp.tools.util.Span span : spans) { + final Span[] spans = segmenter.tokenizePos(text); + for (final Span span : spans) { Assertions.assertEquals(0, span.getStart() % 2, "span must start on a code point boundary: " + span); Assertions.assertEquals(0, span.getEnd() % 2, "span must end on a code point boundary: " + span); } int covered = 0; - for (final opennlp.tools.util.Span span : spans) { + for (final Span span : spans) { covered += span.length(); } Assertions.assertEquals(text.length(), covered); diff --git a/opennlp-docs/src/docbkx/tokenizer.xml b/opennlp-docs/src/docbkx/tokenizer.xml index fbc74fbb2..6f74b6a60 100644 --- a/opennlp-docs/src/docbkx/tokenizer.xml +++ b/opennlp-docs/src/docbkx/tokenizer.xml @@ -542,7 +542,7 @@ wordTokenizer.tokenize("The quick brown fox.", (start, end, type) -> { <title>Lattice tokenization for CJK</title> <para> Languages written without spaces need a dictionary-backed segmenter. - <code>LatticeTokenizer</code> scores paths over a mecab-format dictionary and + <code>LatticeTokenizer</code> scores paths over a MeCab-format dictionary and emits the cheapest segmentation with spans in original text coordinates. <code>UnigramSegmenter</code> does the same from a plain frequency lexicon. Install a dictionary archive with <code>MecabDictionaryInstaller</code>, load it
