This is an automated email from the ASF dual-hosted git repository. krickert pushed a commit to branch OPENNLP-1893-hunspell in repository https://gitbox.apache.org/repos/asf/opennlp.git
commit 3c47dc30591e8b2f7209f2f57526b97cf175df36 Author: Kristian Rickert <[email protected]> AuthorDate: Thu Jul 16 01:43:34 2026 -0400 OPENNLP-1893: Usage, threading, and malformed-input tests for the Hunspell engine, precise javadoc --- .../tools/stemmer/hunspell/AffixCondition.java | 9 +- .../tools/stemmer/hunspell/HunspellDictionary.java | 112 +++++++++++- .../tools/stemmer/hunspell/HunspellStemmer.java | 38 +++- .../stemmer/hunspell/HunspellStemmerFactory.java | 4 + .../hunspell/HunspellStemmerFactoryTest.java | 179 +++++++++++++++++++ .../stemmer/hunspell/HunspellStemmerTest.java | 196 +++++++++++++++++++++ 6 files changed, 522 insertions(+), 16 deletions(-) diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/AffixCondition.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/AffixCondition.java index 8da9a3496..a676b7e48 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/AffixCondition.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/AffixCondition.java @@ -29,6 +29,7 @@ import java.util.List; */ final class AffixCondition { + /** The shared instance for the condition {@code .}, which accepts every stem. */ private static final AffixCondition ANY = new AffixCondition(new char[0][], null, true); /** Per position: the accepted characters, or {@code null} for any character. */ @@ -44,7 +45,9 @@ final class AffixCondition { } /** - * Parses a condition field. + * Parses a condition field. Each pattern position is a literal character, a + * {@code .} matching any character, or a bracketed class such as {@code [sx]}; a + * class starting with {@code ^} is negated and matches any character outside it. * * @param pattern The condition text from the affix rule. * @param suffix Whether the owning rule is a suffix rule. @@ -95,7 +98,9 @@ final class AffixCondition { } /** - * Tests a candidate stem against the condition at its anchored side. + * Tests a candidate stem against the condition at its anchored side: the last + * positions of the stem for a suffix condition, the first positions for a prefix + * condition. A stem shorter than the condition never matches. * * @param stem The candidate stem after affix removal and strip restoration. * @return {@code true} if the stem satisfies the condition. diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellDictionary.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellDictionary.java index 02dff6755..ea8f62339 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellDictionary.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellDictionary.java @@ -54,8 +54,13 @@ import opennlp.tools.util.StringUtil; */ public final class HunspellDictionary { - /** One parsed affix rule; {@code affix} is the surface material added to the stem, - * and {@code continuation} lists the flags of affixes that may stack on top. */ + /** + * One parsed affix rule. {@code affix} is the surface material the rule adds to the + * stem, {@code strip} is the stem material the rule replaces (restored during + * analysis), {@code crossProduct} states whether the rule may combine with an affix + * of the opposite kind, and {@code continuation} lists the flags of further affixes + * that may stack on top of this one. + */ record Affix(int flag, boolean crossProduct, String strip, String affix, AffixCondition condition, int[] continuation) { @@ -169,6 +174,13 @@ public final class HunspellDictionary { return false; } + /** + * Reads a stream fully into memory. The stream is not closed. + * + * @param in The stream to drain. + * @return All bytes the stream produced. Never {@code null}. + * @throws IOException Thrown if reading fails. + */ private static byte[] readAll(InputStream in) throws IOException { final ByteArrayOutputStream out = new ByteArrayOutputStream(); final byte[] buffer = new byte[8192]; @@ -179,7 +191,15 @@ public final class HunspellDictionary { return out.toByteArray(); } - /** Finds the {@code SET} declaration by scanning the raw bytes as ASCII. */ + /** + * Finds the {@code SET} declaration by scanning the raw affix bytes as ASCII, which + * is safe because the declaration itself is ASCII in every supported encoding. Both + * files are then decoded with the declared charset. + * + * @param affixBytes The raw affix file content. + * @return The declared charset, or UTF-8 when no declaration is present. + * @throws IOException Thrown if the declared encoding name is not supported. + */ private static Charset declaredCharset(byte[] affixBytes) throws IOException { final String ascii = new String(affixBytes, StandardCharsets.US_ASCII); for (final String line : splitLines(ascii)) { @@ -196,9 +216,14 @@ public final class HunspellDictionary { return StandardCharsets.UTF_8; } - /** The flag encodings a dictionary may declare. */ + /** The flag encodings a dictionary may declare with the {@code FLAG} directive. */ private enum FlagMode { - CHAR, LONG, NUM + /** The default: each single character is one flag. */ + CHAR, + /** Declared as {@code FLAG long}: each pair of characters is one flag. */ + LONG, + /** Declared as {@code FLAG num}: comma-separated decimal numbers are flags. */ + NUM } /** The parsed affix file content. */ @@ -208,6 +233,16 @@ public final class HunspellDictionary { private FlagMode flagMode = FlagMode.CHAR; } + /** + * Parses the affix file: the {@code FLAG} declaration and the {@code PFX} and + * {@code SFX} blocks. Directives outside the supported set (compounding, + * conversion tables, suggestion options, ...) are skipped, so their rules never + * fire and unsupported analyses are missed rather than invented. + * + * @param content The decoded affix file content. + * @return The parsed rules and flag mode. Never {@code null}. + * @throws IOException Thrown if a supported directive is malformed. + */ private static AffixFile parseAffix(String content) throws IOException { final AffixFile result = new AffixFile(); final String[] lines = splitLines(content); @@ -243,7 +278,18 @@ public final class HunspellDictionary { return result; } - /** Parses one PFX or SFX header and its rule lines; returns the next line index. */ + /** + * Parses one {@code PFX} or {@code SFX} block: the header line naming the flag, the + * cross-product marker, and the rule count, followed by exactly that many rule + * lines. + * + * @param lines All lines of the affix file. + * @param index The line index of the block header. + * @param header The already-split header fields. + * @param result The parse target the rules are added to. + * @return The index of the first line after the block. + * @throws IOException Thrown if the header or a rule line is malformed. + */ private static int parseAffixBlock(String[] lines, int index, String[] header, AffixFile result) throws IOException { if (header.length < 4) { @@ -289,6 +335,17 @@ public final class HunspellDictionary { return line; } + /** + * Parses the word list: an optional leading entry count, then one entry per line + * consisting of the word, an optional {@code /flags} run, and optional + * whitespace-separated morphological fields, which are ignored. A slash escaped as + * {@code \/} belongs to the word itself and is unescaped in the stored key. + * + * @param content The decoded word-list content. + * @param flagMode The flag encoding declared by the affix file. + * @return The words mapped to the flag sets of their entries. Never {@code null}. + * @throws IOException Thrown if a flag run is malformed. + */ private static Map<String, List<int[]>> parseWordList(String content, FlagMode flagMode) throws IOException { final String[] lines = splitLines(content); @@ -325,6 +382,13 @@ public final class HunspellDictionary { return entries; } + /** + * Checks whether a line consists purely of decimal digits, which identifies the + * optional entry-count header of a word list. + * + * @param line The trimmed line to inspect. + * @return {@code true} if the line is a non-empty digit run. + */ private static boolean isCount(String line) { if (line.isEmpty()) { return false; @@ -337,7 +401,13 @@ public final class HunspellDictionary { return true; } - /** Finds the first {@code /} that is not escaped as {@code \/}. */ + /** + * Finds the first {@code /} that is not escaped as {@code \/}, which separates the + * word from its flag run in a word-list entry. + * + * @param line The word-list line to scan. + * @return The index of the separator, or {@code -1} when the entry has no flags. + */ private static int unescapedSlash(String line) { for (int i = 0; i < line.length(); i++) { if (line.charAt(i) == '/' && (i == 0 || line.charAt(i - 1) != '\\')) { @@ -347,6 +417,13 @@ public final class HunspellDictionary { return -1; } + /** + * Finds the first whitespace character, which terminates the word or flag field of + * a word-list entry before its optional morphological fields. + * + * @param text The text to scan. + * @return The index of the first whitespace character, or {@code -1} if none. + */ private static int whitespaceIndex(String text) { for (int i = 0; i < text.length(); i++) { if (StringUtil.isWhitespace(text.charAt(i))) { @@ -356,6 +433,17 @@ public final class HunspellDictionary { return -1; } + /** + * Parses a flag run according to the declared flag mode: single characters in + * {@code char} mode, character pairs packed into one {@code int} in {@code long} + * mode, and comma-separated decimal numbers in {@code num} mode. + * + * @param text The flag run without its leading {@code /}. + * @param mode The declared flag encoding. + * @param lineNumber The source line, for error messages. + * @return The parsed flags. Never {@code null}. + * @throws IOException Thrown if the run does not fit the declared encoding. + */ private static int[] parseFlags(String text, FlagMode mode, int lineNumber) throws IOException { switch (mode) { @@ -391,6 +479,16 @@ public final class HunspellDictionary { } } + /** + * Parses a field that must contain exactly one flag, such as the flag name in an + * affix block header. + * + * @param text The flag field. + * @param mode The declared flag encoding. + * @param lineNumber The source line, for error messages. + * @return The single parsed flag. + * @throws IOException Thrown if the field holds no flag or more than one. + */ private static int parseFlag(String text, FlagMode mode, int lineNumber) throws IOException { final int[] flags = parseFlags(text, mode, lineNumber); diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellStemmer.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellStemmer.java index ba2dd14eb..0639e4ea2 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellStemmer.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellStemmer.java @@ -34,10 +34,12 @@ import opennlp.tools.util.StringUtil; * <p>{@link #stem(CharSequence)} returns the first analysis, preferring the word's own * dictionary entry; {@link #stemAll(CharSequence)} returns every distinct analysis. A * word with no analysis is returned unchanged, so the stemmer degrades to identity on - * unknown vocabulary. Capitalized forms also try their lowercase variant.</p> + * unknown vocabulary. A form containing uppercase characters is also analyzed in its + * lowercase variant, so sentence-initial capitalization does not hide an entry.</p> * - * <p>The stemmer reads only immutable dictionary state and is safe to share between - * threads, satisfying the single-thread confinement contract trivially.</p> + * <p>The {@link Stemmer} interface leaves thread safety to the implementation. This + * implementation reads only the immutable dictionary state, so a single instance is + * safe to share between threads.</p> * * @since 3.0.0 */ @@ -80,13 +82,29 @@ public class HunspellStemmer implements Stemmer { return List.copyOf(new ArrayList<CharSequence>(analyses)); } - /** Collects the case variants to analyze: the surface form, then its lowercase. */ + /** + * Collects the case variants to analyze: the surface form first, then its lowercase + * form when the two differ. Ordering matters because the first analysis found wins + * in {@link #stem(CharSequence)}. + * + * @param surface The surface form. + * @return The variants in analysis order. Never {@code null} or empty. + */ private static List<String> variants(String surface) { final String lowered = StringUtil.toLowerCase(surface); return lowered.equals(surface) ? List.of(surface) : List.of(surface, lowered); } - /** Runs direct lookup, suffix removal, prefix removal, and cross products. */ + /** + * Adds every analysis of one case variant to the result set: the word's own + * dictionary entry, single suffix removal, twofold suffix removal through + * continuation classes, single prefix removal, and cross-product removal of one + * prefix together with one suffix. Insertion order into the set fixes the + * preference order reported by {@link #stemAll(CharSequence)}. + * + * @param word The case variant to analyze. + * @param analyses The mutable, insertion-ordered set collecting the stems found. + */ private void analyze(String word, Set<String> analyses) { if (dictionary.lookup(word) != null) { analyses.add(word); @@ -144,7 +162,10 @@ public class HunspellStemmer implements Stemmer { } /** - * Undoes one suffix rule. + * Undoes one suffix rule: cuts the affix material off the end of the word, restores + * the strip string the rule removed on application, and checks the rule's condition + * against the restored stem. Rules with empty affix material and candidates that + * would leave an empty stem are rejected. * * @param word The surface form. * @param suffix The rule to undo. @@ -161,7 +182,10 @@ public class HunspellStemmer implements Stemmer { } /** - * Undoes one prefix rule. + * Undoes one prefix rule: cuts the affix material off the start of the word, + * restores the strip string the rule removed on application, and checks the rule's + * condition against the restored stem. Rules with empty affix material and + * candidates that would leave an empty stem are rejected. * * @param word The surface form. * @param prefix The rule to undo. diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellStemmerFactory.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellStemmerFactory.java index eb0a8d596..21d66be2b 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellStemmerFactory.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellStemmerFactory.java @@ -45,6 +45,10 @@ public class HunspellStemmerFactory implements StemmerFactory { this.dictionary = dictionary; } + /** + * {@return a new {@link HunspellStemmer} over the shared dictionary} Every call + * creates a fresh instance; all instances read the same immutable dictionary. + */ @Override public Stemmer newStemmer() { return new HunspellStemmer(dictionary); diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/hunspell/HunspellStemmerFactoryTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/hunspell/HunspellStemmerFactoryTest.java new file mode 100644 index 000000000..e19ce1072 --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/hunspell/HunspellStemmerFactoryTest.java @@ -0,0 +1,179 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.stemmer.hunspell; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import opennlp.tools.stemmer.Stemmer; + +/** + * Demonstrates the intended end-to-end usage of the Hunspell stemming classes: a user + * writes (or ships) a {@code .aff}/{@code .dic} file pair, loads it once into a + * {@link HunspellDictionary}, wraps the dictionary in a {@link HunspellStemmerFactory}, + * and obtains {@link Stemmer} instances from the factory wherever stemming is needed. + * The fixture dictionary is authored inside this test class, so no external dictionary + * data is involved. + */ +public class HunspellStemmerFactoryTest { + + /** + * The affix fixture: the prefix {@code re-}, the suffix {@code -er} whose continuation + * class {@code S} lets the plural {@code -s} stack on top of it, and the plural + * {@code -s} itself, restricted to stems not ending in {@code s}, {@code x}, or + * {@code y}. All three rules opt into cross-product combination. + */ + private static final String AFFIX = String.join("\n", + "# project-authored test fixture", + "SET UTF-8", + "", + "PFX R Y 1", + "PFX R 0 re .", + "", + "SFX E Y 1", + "SFX E 0 er/S .", + "", + "SFX S Y 1", + "SFX S 0 s [^sxy]", + ""); + + /** + * The word-list fixture: {@code work} accepts the prefix and both suffixes, + * {@code paint} accepts only the agentive {@code -er}. + */ + private static final String WORDS = String.join("\n", + "2", + "work/RES", + "paint/E", + ""); + + /** + * Writes the fixture dictionary pair into a directory and loads it through the + * file-based {@link HunspellDictionary#load(Path, Path)} entry point. + * + * @param directory The directory to write into. Must not be {@code null} and must + * denote an existing directory. + * @return The loaded dictionary. Never {@code null}. + * @throws IOException Thrown if writing or loading fails. + * @throws IllegalArgumentException Thrown if {@code directory} is unusable. + */ + private static HunspellDictionary writeAndLoadFixture(Path directory) throws IOException { + if (directory == null || !Files.isDirectory(directory)) { + throw new IllegalArgumentException("directory must be an existing directory"); + } + final Path affixFile = directory.resolve("fixture.aff"); + final Path dictionaryFile = directory.resolve("fixture.dic"); + Files.write(affixFile, AFFIX.getBytes(StandardCharsets.UTF_8)); + Files.write(dictionaryFile, WORDS.getBytes(StandardCharsets.UTF_8)); + return HunspellDictionary.load(affixFile, dictionaryFile); + } + + /** + * Walks the whole intended flow on a single thread: files on disk, one dictionary, + * one factory, one stemmer, and exact stems for a prefixed form, a suffixed form, a + * twofold suffix chain, a cross-product form, an in-dictionary word, and an unknown + * word. + * + * @param tempDir A scratch directory managed by the test framework. + * @throws IOException Thrown if the fixture cannot be written or loaded. + */ + @Test + void testEndToEndUsageFromFiles(@TempDir Path tempDir) throws IOException { + final HunspellDictionary dictionary = writeAndLoadFixture(tempDir); + final HunspellStemmerFactory factory = new HunspellStemmerFactory(dictionary); + final Stemmer stemmer = factory.newStemmer(); + + // one suffix removed + Assertions.assertEquals("work", stemmer.stem("worker").toString()); + Assertions.assertEquals("paint", stemmer.stem("painter").toString()); + // twofold suffixes: -s stacks on -er through the continuation class S + Assertions.assertEquals("work", stemmer.stem("workers").toString()); + // one prefix removed + Assertions.assertEquals("work", stemmer.stem("rework").toString()); + // cross product: the prefix re- and the suffix -s on the same stem + Assertions.assertEquals("work", stemmer.stem("reworks").toString()); + // a word that is itself listed stems to itself + Assertions.assertEquals("work", stemmer.stem("work").toString()); + // unknown vocabulary passes through unchanged + Assertions.assertEquals("table", stemmer.stem("table").toString()); + } + + /** + * Shares one factory between two threads: each thread obtains its own stemmer + * instance from the factory and stems the same inputs. The test asserts that the two + * instances are distinct objects and that their results are identical to each other + * and to the expected stems. + * + * @param tempDir A scratch directory managed by the test framework. + * @throws Exception Thrown if the fixture cannot be loaded or a worker fails. + */ + @Test + void testFactorySharedAcrossThreads(@TempDir Path tempDir) throws Exception { + final HunspellStemmerFactory factory = + new HunspellStemmerFactory(writeAndLoadFixture(tempDir)); + final List<String> inputs = List.of("workers", "reworks", "painter", "table"); + final List<String> expected = List.of("work", "work", "paint", "table"); + + final Stemmer[] created = new Stemmer[2]; + final ExecutorService pool = Executors.newFixedThreadPool(2); + try { + final List<Future<List<String>>> futures = new ArrayList<>(2); + for (int worker = 0; worker < 2; worker++) { + final int slot = worker; + futures.add(pool.submit(() -> { + final Stemmer stemmer = factory.newStemmer(); + created[slot] = stemmer; + final List<String> stems = new ArrayList<>(inputs.size()); + for (final String input : inputs) { + stems.add(stemmer.stem(input).toString()); + } + return stems; + })); + } + final List<String> first = futures.get(0).get(); + final List<String> second = futures.get(1).get(); + Assertions.assertEquals(expected, first); + Assertions.assertEquals(expected, second); + } finally { + pool.shutdownNow(); + } + Assertions.assertNotSame(created[0], created[1]); + } + + /** + * Verifies that the file-based entry point rejects {@code null} paths with the + * documented exception instead of failing later with an obscure error. + */ + @Test + void testNullPathsAreRejected() { + final IllegalArgumentException e = Assertions.assertThrows(IllegalArgumentException.class, + () -> HunspellDictionary.load((Path) null, (Path) null)); + Assertions.assertEquals("affixFile and dictionaryFile must not be null", e.getMessage()); + } +} diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/hunspell/HunspellStemmerTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/hunspell/HunspellStemmerTest.java index 2045f6e64..c6304ab15 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/hunspell/HunspellStemmerTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/hunspell/HunspellStemmerTest.java @@ -19,6 +19,7 @@ package opennlp.tools.stemmer.hunspell; import java.io.ByteArrayInputStream; import java.io.IOException; +import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; import org.junit.jupiter.api.Assertions; @@ -179,6 +180,201 @@ public class HunspellStemmerTest { Assertions.assertEquals("pony", fresh.stem("ponies").toString()); } + /** + * Loads a dictionary from in-memory affix and word-list content, both encoded as + * UTF-8, through the stream-based entry point. + * + * @param affix The {@code .aff} content. Must not be {@code null}. + * @param words The {@code .dic} content. Must not be {@code null}. + * @return The loaded dictionary. Never {@code null}. + * @throws IOException Thrown if the content is malformed. + * @throws IllegalArgumentException Thrown if a parameter is {@code null}. + */ + private static HunspellDictionary load(String affix, String words) throws IOException { + if (affix == null || words == null) { + throw new IllegalArgumentException("affix and words must not be null"); + } + return HunspellDictionary.load( + new ByteArrayInputStream(affix.getBytes(StandardCharsets.UTF_8)), + new ByteArrayInputStream(words.getBytes(StandardCharsets.UTF_8))); + } + + /** + * Verifies that cross-product combination of a prefix with a suffix only happens + * when both rules declare the cross-product marker {@code Y}. Removing just the one + * affix whose rule exists keeps working; the combined form must not be analyzed. + * + * @throws IOException Thrown if a fixture fails to load. + */ + @Test + void testCrossProductRequiresBothRulesOptIn() throws IOException { + // the prefix rule declares N, so it never combines with the suffix + final HunspellStemmer prefixOptedOut = new HunspellStemmer(load(String.join("\n", + "PFX U N 1", + "PFX U 0 un .", + "SFX S Y 1", + "SFX S 0 s .", + ""), "1\nlock/US\n")); + Assertions.assertEquals("lock", prefixOptedOut.stem("unlock").toString()); + Assertions.assertEquals("lock", prefixOptedOut.stem("locks").toString()); + Assertions.assertEquals("unlocks", prefixOptedOut.stem("unlocks").toString()); + + // the suffix rule declares N, so the combined form is likewise not analyzed + final HunspellStemmer suffixOptedOut = new HunspellStemmer(load(String.join("\n", + "PFX U Y 1", + "PFX U 0 un .", + "SFX S N 1", + "SFX S 0 s .", + ""), "1\nlock/US\n")); + Assertions.assertEquals("lock", suffixOptedOut.stem("unlock").toString()); + Assertions.assertEquals("lock", suffixOptedOut.stem("locks").toString()); + Assertions.assertEquals("unlocks", suffixOptedOut.stem("unlocks").toString()); + } + + /** + * Verifies that a non-negated character class rejects a candidate stem: the + * {@code es} rule requires a stem ending in {@code s} or {@code x}, so removing + * {@code es} from {@code cates} produces {@code cat}, which the class rejects, and + * the surface form falls through unchanged. + */ + @Test + void testPositiveCharacterClassRejectsCandidate() { + Assertions.assertEquals("cates", stemmer.stem("cates").toString()); + Assertions.assertEquals(1, stemmer.stemAll("cates").size()); + } + + /** + * Verifies that the {@code SET} declaration selects the charset both files are + * decoded with: a word list holding the byte {@code 0xE9} only maps to the word + * caf\u00E9 (e with acute accent) when decoded as ISO-8859-1, as the affix file declares. + * + * @throws IOException Thrown if the fixture fails to load. + */ + @Test + void testSetDeclarationSelectsEncoding() throws IOException { + final Charset latin1 = StandardCharsets.ISO_8859_1; + final String affix = String.join("\n", + "SET ISO8859-1", + "SFX S Y 1", + "SFX S 0 s .", + ""); + final String words = "1\ncaf\u00E9/S\n"; + final HunspellDictionary dictionary = HunspellDictionary.load( + new ByteArrayInputStream(affix.getBytes(latin1)), + new ByteArrayInputStream(words.getBytes(latin1))); + final HunspellStemmer latin1Stemmer = new HunspellStemmer(dictionary); + Assertions.assertEquals("caf\u00E9", latin1Stemmer.stem("caf\u00E9s").toString()); + Assertions.assertEquals("caf\u00E9", latin1Stemmer.stem("caf\u00E9").toString()); + } + + /** + * Verifies that continuation classes also work in {@code FLAG long} mode, where a + * flag is a two-character run: the plural {@code Bb} stacks on the agentive + * {@code Aa} to analyze a twofold suffix chain. + * + * @throws IOException Thrown if the fixture fails to load. + */ + @Test + void testLongFlagContinuation() throws IOException { + final HunspellStemmer longFlags = new HunspellStemmer(load(String.join("\n", + "FLAG long", + "SFX Aa Y 1", + "SFX Aa 0 er/Bb .", + "SFX Bb Y 1", + "SFX Bb 0 s .", + ""), "1\nkind/Aa\n")); + Assertions.assertEquals("kind", longFlags.stem("kinder").toString()); + Assertions.assertEquals("kind", longFlags.stem("kinders").toString()); + } + + /** + * Verifies that cross-product prefix and suffix combination also works in + * {@code FLAG num} mode, where flags are comma-separated decimal numbers. + * + * @throws IOException Thrown if the fixture fails to load. + */ + @Test + void testNumericFlagCrossProduct() throws IOException { + final HunspellStemmer numericFlags = new HunspellStemmer(load(String.join("\n", + "FLAG num", + "PFX 1 Y 1", + "PFX 1 0 un .", + "SFX 2 Y 1", + "SFX 2 0 s .", + ""), "1\nlock/1,2\n")); + Assertions.assertEquals("lock", numericFlags.stem("unlock").toString()); + Assertions.assertEquals("lock", numericFlags.stem("locks").toString()); + Assertions.assertEquals("lock", numericFlags.stem("unlocks").toString()); + } + + /** + * Verifies the exact exception and message for each malformed {@code FLAG} + * declaration the parser detects: a missing mode and an unrecognized mode name. + */ + @Test + void testMalformedFlagDeclarationMessages() { + IOException e = Assertions.assertThrows(IOException.class, + () -> load("FLAG\n", "0\n")); + Assertions.assertEquals("FLAG line without a mode at line 1", e.getMessage()); + + e = Assertions.assertThrows(IOException.class, () -> load("FLAG short\n", "0\n")); + Assertions.assertEquals("unsupported FLAG mode 'short' at line 1", e.getMessage()); + } + + /** + * Verifies the exact exception and message for each malformed affix block the + * parser detects: a header with too few fields, a non-numeric rule count, a block + * with fewer rule lines than its count announces, a rule line whose type tag does + * not match its header, and an unterminated character class in a condition. + */ + @Test + void testMalformedAffixBlockMessages() { + IOException e = Assertions.assertThrows(IOException.class, + () -> load("PFX U Y\n", "0\n")); + Assertions.assertEquals("malformed affix header at line 1", e.getMessage()); + + e = Assertions.assertThrows(IOException.class, + () -> load("SFX S Y many\nSFX S 0 s .\n", "0\n")); + Assertions.assertEquals("malformed affix rule count at line 1", e.getMessage()); + + e = Assertions.assertThrows(IOException.class, + () -> load("SFX S Y 2\nSFX S 0 s .", "0\n")); + Assertions.assertEquals("affix block truncated at line 3", e.getMessage()); + + e = Assertions.assertThrows(IOException.class, + () -> load("SFX S Y 1\nPFX S 0 s .\n", "0\n")); + Assertions.assertEquals("malformed affix rule at line 2", e.getMessage()); + + e = Assertions.assertThrows(IOException.class, + () -> load("SFX S Y 1\nSFX S 0 s [ab\n", "0\n")); + Assertions.assertEquals("unterminated character class at line 2", e.getMessage()); + } + + /** + * Verifies the exact exception and message for each malformed flag value the parser + * detects: an odd-length flag run in {@code FLAG long} mode, a non-numeric flag in + * {@code FLAG num} mode, an affix header naming more than one flag, and a + * {@code SET} declaration naming an unknown encoding. + */ + @Test + void testMalformedFlagValueMessages() { + IOException e = Assertions.assertThrows(IOException.class, + () -> load("FLAG long\n", "1\nwalk/AaB\n")); + Assertions.assertEquals("odd long-flag run at line 2", e.getMessage()); + + e = Assertions.assertThrows(IOException.class, + () -> load("FLAG num\n", "1\nwalk/12,x\n")); + Assertions.assertEquals("malformed numeric flag at line 2", e.getMessage()); + + e = Assertions.assertThrows(IOException.class, + () -> load("FLAG long\nSFX AaBb Y 1\nSFX AaBb 0 s .\n", "0\n")); + Assertions.assertEquals("expected exactly one flag at line 2", e.getMessage()); + + e = Assertions.assertThrows(IOException.class, + () -> load("SET NO-SUCH-ENCODING\n", "0\n")); + Assertions.assertEquals("unsupported SET encoding: NO-SUCH-ENCODING", e.getMessage()); + } + @Test void testMalformedInputFailsLoud() { Assertions.assertThrows(IOException.class, () -> HunspellDictionary.load(
