This is an automated email from the ASF dual-hosted git repository. krickert pushed a commit to branch OPENNLP-1893-hunspell in repository https://gitbox.apache.org/repos/asf/opennlp.git
commit 9cf76647b0cff2b626213f46a3fcbadfd036aaa9 Author: Kristian Rickert <[email protected]> AuthorDate: Thu Jul 16 22:36:16 2026 -0400 OPENNLP-1893: Cut morphology like hunspell does, accept UTF-8 flags and strip-only rules, and read the parser through the whitespace seam --- .../dev/README-hunspell-dictionaries.md | 4 +- .../tools/stemmer/hunspell/HunspellDictionary.java | 109 +++++++++++----- .../tools/stemmer/hunspell/HunspellStemmer.java | 27 ++-- .../stemmer/hunspell/HunspellStemmerTest.java | 144 +++++++++++++++++++++ 4 files changed, 237 insertions(+), 47 deletions(-) diff --git a/opennlp-core/opennlp-runtime/dev/README-hunspell-dictionaries.md b/opennlp-core/opennlp-runtime/dev/README-hunspell-dictionaries.md index 98195b029..31edf4fe0 100644 --- a/opennlp-core/opennlp-runtime/dev/README-hunspell-dictionaries.md +++ b/opennlp-core/opennlp-runtime/dev/README-hunspell-dictionaries.md @@ -46,8 +46,10 @@ Stemmer stemmer = factory.newStemmer(); CharSequence stem = stemmer.stem("workers"); ``` +What `stem` evaluates to is decided by the dictionary you loaded, and this project ships no dictionary data, so no result is claimed here for `en_US`. What is verified is the flow above: `HunspellStemmerFactoryTest#testEndToEndUsageFromFiles` runs exactly these calls against a project-authored `.aff`/`.dic` pair written to disk, in which `work` carries the agentive `-er` rule and its continuation class for the plural `-s`, and asserts that `stemmer.stem("workers")` returns `work`. + The dictionary is immutable and safe to share between threads; the factory hands out a fresh stemmer per call, so each thread takes its own from `newStemmer()`. A dictionary that declares a non-UTF-8 encoding through the `SET` directive in its `.aff` file is decoded accordingly; nothing needs converting beforehand. ## What the engine supports -Supported affix features: `PFX` and `SFX` rules with strip strings, character-class conditions, cross-product combination of one prefix with one suffix, twofold suffixes through continuation classes, `FLAG` modes `char`, `long`, and `num`, and the `SET` encoding declaration. Compounding and conversion tables are not interpreted; rules that use them simply do not fire, so unsupported analyses are missed rather than invented. A malformed `.aff` file fails loudly at load time with the offen [...] +Supported affix features: `PFX` and `SFX` rules with strip strings, character-class conditions, cross-product combination of one prefix with one suffix, twofold suffixes through continuation classes, `FLAG` modes `char`, `UTF-8`, `long`, and `num`, and the `SET` encoding declaration. Compounding and conversion tables are not interpreted; rules that use them simply do not fire, so unsupported analyses are missed rather than invented. A malformed `.aff` file fails loudly at load time with [...] diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellDictionary.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellDictionary.java index ea8f62339..6dbb001eb 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellDictionary.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellDictionary.java @@ -41,8 +41,9 @@ import opennlp.tools.util.StringUtil; * <p>Supported affix features: {@code PFX} and {@code SFX} rules with strip strings, * character-class conditions, and cross-product combination of one prefix with one * suffix; twofold suffixes through the continuation classes on suffix rules; - * {@code FLAG} modes {@code char} (default), {@code long}, and {@code num}; the - * {@code SET} encoding declaration. Compounding and conversion tables are not + * {@code FLAG} modes {@code char} (default), {@code UTF-8}, {@code long}, and + * {@code num}; the {@code SET} encoding declaration. Compounding and conversion tables + * are not * interpreted in this version; rules using them simply do not fire, so unsupported * analyses are missed rather than invented.</p> * @@ -203,9 +204,9 @@ public final class HunspellDictionary { private static Charset declaredCharset(byte[] affixBytes) throws IOException { final String ascii = new String(affixBytes, StandardCharsets.US_ASCII); for (final String line : splitLines(ascii)) { - final String trimmed = line.trim(); + final String trimmed = trim(line); if (trimmed.startsWith("SET ") || trimmed.startsWith("SET\t")) { - final String name = trimmed.substring(4).trim(); + final String name = trim(trimmed.substring(4)); try { return Charset.forName(name); } catch (RuntimeException e) { @@ -218,7 +219,11 @@ public final class HunspellDictionary { /** The flag encodings a dictionary may declare with the {@code FLAG} directive. */ private enum FlagMode { - /** The default: each single character is one flag. */ + /** + * The default: each single character is one flag. Also what {@code FLAG UTF-8} + * declares, which asks for single-character flags in a file the {@code SET} + * declaration already had decoded. + */ CHAR, /** Declared as {@code FLAG long}: each pair of characters is one flag. */ LONG, @@ -261,6 +266,7 @@ public final class HunspellDictionary { result.flagMode = switch (fields[1]) { case "long" -> FlagMode.LONG; case "num" -> FlagMode.NUM; + case "UTF-8" -> FlagMode.CHAR; default -> throw new IOException( "unsupported FLAG mode '" + fields[1] + "' at line " + (i + 1)); }; @@ -337,9 +343,11 @@ public final class HunspellDictionary { /** * Parses the word list: an optional leading entry count, then one entry per line - * consisting of the word, an optional {@code /flags} run, and optional - * whitespace-separated morphological fields, which are ignored. A slash escaped as - * {@code \/} belongs to the word itself and is unescaped in the stored key. + * consisting of the word, an optional {@code /flags} run, and optional trailing + * morphological fields, which are ignored. The morphological fields are cut off + * first, because the flag separator is only meaningful in what precedes them; a word + * may itself contain spaces. A slash escaped as {@code \/} belongs to the word itself + * and is unescaped in the stored key. * * @param content The decoded word-list content. * @param flagMode The flag encoding declared by the affix file. @@ -351,30 +359,22 @@ public final class HunspellDictionary { final String[] lines = splitLines(content); final Map<String, List<int[]>> entries = new HashMap<>(); int start = 0; - if (lines.length > 0 && isCount(lines[0].trim())) { + if (lines.length > 0 && isCount(trim(lines[0]))) { start = 1; } for (int i = start; i < lines.length; i++) { - final String line = lines[i].trim(); + final String line = trim(lines[i]); if (line.isEmpty()) { continue; } - String word = line; + final int morphology = morphologyIndex(line); + final String entry = morphology < 0 ? line : trim(line.substring(0, morphology)); + String word = entry; int[] flags = new int[0]; - final int slash = unescapedSlash(line); + final int slash = unescapedSlash(entry); if (slash >= 0) { - word = line.substring(0, slash); - String flagText = line.substring(slash + 1); - final int fieldEnd = whitespaceIndex(flagText); - if (fieldEnd >= 0) { - flagText = flagText.substring(0, fieldEnd); - } - flags = parseFlags(flagText, flagMode, i + 1); - } else { - final int fieldEnd = whitespaceIndex(word); - if (fieldEnd >= 0) { - word = word.substring(0, fieldEnd); - } + word = entry.substring(0, slash); + flags = parseFlags(entry.substring(slash + 1), flagMode, i + 1); } entries.computeIfAbsent(word.replace("\\/", "/"), key -> new ArrayList<>(1)) .add(flags); @@ -418,19 +418,54 @@ public final class HunspellDictionary { } /** - * Finds the first whitespace character, which terminates the word or flag field of - * a word-list entry before its optional morphological fields. + * Finds where the trailing morphological fields of a word-list entry begin, which + * terminates the word and its flag run. A morphological field is either introduced by + * a tabulator, the older separator, or written as a two-letter tag followed by + * {@code :} and preceded by whitespace, such as {@code po:verb}. Whitespace that is + * not followed by such a tag belongs to the word, because a word-list entry may name + * several words. * - * @param text The text to scan. - * @return The index of the first whitespace character, or {@code -1} if none. + * @param line The trimmed word-list line to scan. + * @return The index at which the morphological fields begin, or {@code -1} if the + * entry carries none. */ - private static int whitespaceIndex(String text) { - for (int i = 0; i < text.length(); i++) { - if (StringUtil.isWhitespace(text.charAt(i))) { - return i; + private static int morphologyIndex(String line) { + int cut = -1; + for (int i = 4; i < line.length(); i++) { + if (line.charAt(i) == ':' && StringUtil.isWhitespace(line.charAt(i - 3))) { + int fieldStart = i - 3; + while (fieldStart > 0 && StringUtil.isWhitespace(line.charAt(fieldStart - 1))) { + fieldStart--; + } + // a tag with no word in front of it is not a morphological field + cut = fieldStart == 0 ? -1 : fieldStart; + break; } } - return -1; + final int tab = line.indexOf('\t'); + if (tab >= 0 && (cut < 0 || tab < cut)) { + cut = tab; + } + return cut; + } + + /** + * Removes leading and trailing whitespace, using the whitespace definition the rest + * of the parser scans with. + * + * @param text The text to trim. + * @return The text without leading or trailing whitespace. Never {@code null}. + */ + private static String trim(String text) { + int start = 0; + int end = text.length(); + while (start < end && StringUtil.isWhitespace(text.charAt(start))) { + start++; + } + while (end > start && StringUtil.isWhitespace(text.charAt(end - 1))) { + end--; + } + return text.substring(start, end); } /** @@ -438,7 +473,8 @@ public final class HunspellDictionary { * {@code char} mode, character pairs packed into one {@code int} in {@code long} * mode, and comma-separated decimal numbers in {@code num} mode. * - * @param text The flag run without its leading {@code /}. + * @param text The flag run without its leading {@code /}. An empty run carries no + * flags in every mode. * @param mode The declared flag encoding. * @param lineNumber The source line, for error messages. * @return The parsed flags. Never {@code null}. @@ -446,13 +482,16 @@ public final class HunspellDictionary { */ private static int[] parseFlags(String text, FlagMode mode, int lineNumber) throws IOException { + if (text.isEmpty()) { + return new int[0]; + } switch (mode) { case NUM: { final String[] parts = splitOn(text, ','); final int[] flags = new int[parts.length]; for (int i = 0; i < parts.length; i++) { try { - flags[i] = Integer.parseInt(parts[i].trim()); + flags[i] = Integer.parseInt(trim(parts[i])); } catch (NumberFormatException e) { throw new IOException("malformed numeric flag at line " + lineNumber, e); } diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellStemmer.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellStemmer.java index 0639e4ea2..eb55bb6d8 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellStemmer.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellStemmer.java @@ -164,39 +164,44 @@ public class HunspellStemmer implements Stemmer { /** * Undoes one suffix rule: cuts the affix material off the end of the word, restores * the strip string the rule removed on application, and checks the rule's condition - * against the restored stem. Rules with empty affix material and candidates that - * would leave an empty stem are rejected. + * against the restored stem. A strip-only rule, whose affix material is empty, is + * undone by restoring its strip string alone. Rules that neither add nor remove + * material and candidates that would leave an empty stem are rejected. * * @param word The surface form. * @param suffix The rule to undo. * @return The candidate stem, or {@code null} when the rule does not apply. */ private static String removeSuffix(String word, Affix suffix) { - if (suffix.affix().isEmpty() || !word.endsWith(suffix.affix()) - || word.length() - suffix.affix().length() + suffix.strip().length() == 0) { + final String affix = suffix.affix(); + final String strip = suffix.strip(); + if (affix.isEmpty() && strip.isEmpty() || !word.endsWith(affix) + || word.length() - affix.length() + strip.length() == 0) { return null; } - final String stem = - word.substring(0, word.length() - suffix.affix().length()) + suffix.strip(); + final String stem = word.substring(0, word.length() - affix.length()) + strip; return suffix.condition().matches(stem) ? stem : null; } /** * Undoes one prefix rule: cuts the affix material off the start of the word, * restores the strip string the rule removed on application, and checks the rule's - * condition against the restored stem. Rules with empty affix material and - * candidates that would leave an empty stem are rejected. + * condition against the restored stem. A strip-only rule, whose affix material is + * empty, is undone by restoring its strip string alone. Rules that neither add nor + * remove material and candidates that would leave an empty stem are rejected. * * @param word The surface form. * @param prefix The rule to undo. * @return The candidate stem, or {@code null} when the rule does not apply. */ private static String removePrefix(String word, Affix prefix) { - if (prefix.affix().isEmpty() || !word.startsWith(prefix.affix()) - || word.length() - prefix.affix().length() + prefix.strip().length() == 0) { + final String affix = prefix.affix(); + final String strip = prefix.strip(); + if (affix.isEmpty() && strip.isEmpty() || !word.startsWith(affix) + || word.length() - affix.length() + strip.length() == 0) { return null; } - final String stem = prefix.strip() + word.substring(prefix.affix().length()); + final String stem = strip + word.substring(affix.length()); return prefix.condition().matches(stem) ? stem : null; } } diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/hunspell/HunspellStemmerTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/hunspell/HunspellStemmerTest.java index c6304ab15..82956bd9b 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/hunspell/HunspellStemmerTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/hunspell/HunspellStemmerTest.java @@ -375,6 +375,150 @@ public class HunspellStemmerTest { Assertions.assertEquals("unsupported SET encoding: NO-SUCH-ENCODING", e.getMessage()); } + /** + * Verifies that a morphological field is cut off the entry before the flag separator + * is looked for, so a slash inside a morphological field is not mistaken for the + * separator: the entry {@code walk po:verb/noun} registers the word {@code walk} + * with no flags in every flag mode, and its morphology is ignored. + * + * @throws IOException Thrown if a fixture fails to load. + */ + @Test + void testMorphologicalFieldsAreCutBeforeTheFlagSeparator() throws IOException { + final HunspellDictionary chars = load("SFX G Y 1\nSFX G 0 ing .\n", + "1\nwalk po:verb/noun\n"); + Assertions.assertNotNull(chars.lookup("walk")); + Assertions.assertEquals(0, chars.lookup("walk").get(0).length); + Assertions.assertNull(chars.lookup("walk po:verb")); + + final HunspellDictionary numbers = load("FLAG num\nSFX 1 Y 1\nSFX 1 0 ing .\n", + "1\nwalk po:verb/noun\n"); + Assertions.assertNotNull(numbers.lookup("walk")); + Assertions.assertEquals(0, numbers.lookup("walk").get(0).length); + + // the tabulator is the older morphological field separator + final HunspellDictionary tabbed = load("SFX G Y 1\nSFX G 0 ing .\n", + "1\nwalk\tpo:verb/noun\n"); + Assertions.assertNotNull(tabbed.lookup("walk")); + Assertions.assertEquals(0, tabbed.lookup("walk").get(0).length); + } + + /** + * Verifies that an entry keeps its flags when it carries both a flag run and a + * morphological field holding a slash, in every flag mode. + * + * @throws IOException Thrown if a fixture fails to load. + */ + @Test + void testFlaggedEntriesKeepTheirFlagsBesideMorphology() throws IOException { + final HunspellDictionary chars = load("SFX A Y 1\nSFX A 0 ing .\n", + "1\nwalk/AB po:verb/noun\n"); + Assertions.assertArrayEquals(new int[] {'A', 'B'}, chars.lookup("walk").get(0)); + Assertions.assertEquals("walk", + new HunspellStemmer(chars).stem("walking").toString()); + + final HunspellDictionary numbers = load("FLAG num\nSFX 1 Y 1\nSFX 1 0 ing .\n", + "1\nwalk/1,2 po:verb/noun\n"); + Assertions.assertArrayEquals(new int[] {1, 2}, numbers.lookup("walk").get(0)); + Assertions.assertEquals("walk", + new HunspellStemmer(numbers).stem("walking").toString()); + } + + /** + * Verifies that a multi-word entry keeps both its spaces and its flags: the word of + * a word-list entry runs up to its morphological fields, not up to its first space. + * + * @throws IOException Thrown if the fixture fails to load. + */ + @Test + void testMultiWordEntriesKeepTheirSpacesAndFlags() throws IOException { + final HunspellDictionary dictionary = load("FLAG num\nSFX 39 Y 1\nSFX 39 0 s .\n", + "1\nall right/39\n"); + Assertions.assertArrayEquals(new int[] {39}, dictionary.lookup("all right").get(0)); + Assertions.assertNull(dictionary.lookup("all")); + } + + /** + * Verifies that the parser trims word-list entries with the same whitespace + * definition it uses to find their fields: an entry led by a no-break space is + * registered under its real word, both with and without a flag run. + * + * @throws IOException Thrown if a fixture fails to load. + */ + @Test + void testEntriesLedByNoBreakSpaceAreTrimmed() throws IOException { + // \u00A0 is the no-break space, which StringUtil.isWhitespace treats as whitespace + final HunspellDictionary dictionary = load("SFX S Y 1\nSFX S 0 s .\n", + "2\n\u00A0fish\n\u00A0cat/S\n"); + Assertions.assertNotNull(dictionary.lookup("fish")); + Assertions.assertNotNull(dictionary.lookup("cat")); + Assertions.assertNull(dictionary.lookup("")); + Assertions.assertEquals("cat", new HunspellStemmer(dictionary).stem("cats").toString()); + } + + /** + * Verifies that {@code FLAG UTF-8}, which declares single-character flags, is + * accepted and read exactly like the default single-character mode, including a flag + * outside ASCII. + * + * @throws IOException Thrown if a fixture fails to load. + */ + @Test + void testUtf8FlagModeDeclaresSingleCharacterFlags() throws IOException { + final HunspellStemmer plain = new HunspellStemmer(load(String.join("\n", + "FLAG UTF-8", + "SFX S Y 1", + "SFX S 0 s .", + ""), "1\nwalk/S\n")); + Assertions.assertEquals("walk", plain.stem("walks").toString()); + + // \u00E9 is e with an acute accent, a single-character flag outside ASCII + final HunspellStemmer accented = new HunspellStemmer(load(String.join("\n", + "FLAG UTF-8", + "SFX \u00E9 Y 1", + "SFX \u00E9 0 s .", + ""), "1\nwalk/\u00E9\n")); + Assertions.assertEquals("walk", accented.stem("walks").toString()); + } + + /** + * Verifies that a strip-only rule, whose affix material is empty and which therefore + * only removes stem material, is undone: the suffix rule turns the entry + * {@code bake} into the surface form {@code bak}, and the prefix rule turns + * {@code apple} into {@code pple}. + * + * @throws IOException Thrown if a fixture fails to load. + */ + @Test + void testStripOnlyAffixRulesAreUndone() throws IOException { + final HunspellStemmer suffixStripping = new HunspellStemmer(load(String.join("\n", + "SFX A Y 1", + "SFX A e 0 e", + ""), "1\nbake/A\n")); + Assertions.assertEquals("bake", suffixStripping.stem("bak").toString()); + + final HunspellStemmer prefixStripping = new HunspellStemmer(load(String.join("\n", + "PFX B Y 1", + "PFX B a 0 a", + ""), "1\napple/B\n")); + Assertions.assertEquals("apple", prefixStripping.stem("pple").toString()); + } + + /** + * Verifies that an entry written with an empty flag run loads and carries no flags in + * every flag mode, rather than failing the load in {@code FLAG num} mode alone. + * + * @throws IOException Thrown if a fixture fails to load. + */ + @Test + void testEmptyFlagRunYieldsNoFlagsInEveryMode() throws IOException { + Assertions.assertEquals(0, load("", "1\nword/\n").lookup("word").get(0).length); + Assertions.assertEquals(0, + load("FLAG long\n", "1\nword/\n").lookup("word").get(0).length); + Assertions.assertEquals(0, + load("FLAG num\n", "1\nword/\n").lookup("word").get(0).length); + } + @Test void testMalformedInputFailsLoud() { Assertions.assertThrows(IOException.class, () -> HunspellDictionary.load(
