This is an automated email from the ASF dual-hosted git repository. krickert pushed a commit to branch OPENNLP-1893-hunspell in repository https://gitbox.apache.org/repos/asf/opennlp.git
commit f85e909d3b7f9d2c5d49c67c7eb3c37a29280dc7 Author: Kristian Rickert <[email protected]> AuthorDate: Wed Jul 15 13:53:05 2026 -0400 OPENNLP-1893: Twofold suffix analysis through Hunspell continuation classes Suffix rules now carry the continuation flags declared on their affix text, and analysis undoes a stacked pair when the inner rule's classes allow the outer one, so derived-then-inflected forms reduce to their dictionary word. (cherry picked from commit b543dea24c5241d5f3431b34be987f6e83fe887e) --- .../tools/stemmer/hunspell/HunspellDictionary.java | 39 ++++++++++++++++------ .../tools/stemmer/hunspell/HunspellStemmer.java | 13 ++++++++ .../stemmer/hunspell/HunspellStemmerTest.java | 21 ++++++++++++ 3 files changed, 63 insertions(+), 10 deletions(-) diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellDictionary.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellDictionary.java index 8ac526fc3..02dff6755 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellDictionary.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellDictionary.java @@ -40,10 +40,11 @@ import opennlp.tools.util.StringUtil; * * <p>Supported affix features: {@code PFX} and {@code SFX} rules with strip strings, * character-class conditions, and cross-product combination of one prefix with one - * suffix; {@code FLAG} modes {@code char} (default), {@code long}, and {@code num}; - * the {@code SET} encoding declaration. Compounding, continuation classes, and - * conversion tables are not interpreted in this version; rules using them simply do - * not fire, so unsupported analyses are missed rather than invented.</p> + * suffix; twofold suffixes through the continuation classes on suffix rules; + * {@code FLAG} modes {@code char} (default), {@code long}, and {@code num}; the + * {@code SET} encoding declaration. Compounding and conversion tables are not + * interpreted in this version; rules using them simply do not fire, so unsupported + * analyses are missed rather than invented.</p> * * <p>Instances are immutable and safe to share between threads.</p> * @@ -53,9 +54,25 @@ import opennlp.tools.util.StringUtil; */ public final class HunspellDictionary { - /** One parsed affix rule; {@code affix} is the surface material added to the stem. */ + /** One parsed affix rule; {@code affix} is the surface material added to the stem, + * and {@code continuation} lists the flags of affixes that may stack on top. */ record Affix(int flag, boolean crossProduct, String strip, String affix, - AffixCondition condition) { + AffixCondition condition, int[] continuation) { + + /** + * Checks whether a further affix may stack on this one. + * + * @param otherFlag The stacking affix's flag. + * @return {@code true} if this affix's continuation classes allow it. + */ + boolean allowsContinuation(int otherFlag) { + for (final int candidate : continuation) { + if (candidate == otherFlag) { + return true; + } + } + return false; + } } private final Map<String, List<int[]>> entries; @@ -252,15 +269,17 @@ public final class HunspellDictionary { } final String strip = "0".equals(fields[2]) ? "" : fields[2]; String affixText = fields[3]; - final int continuation = affixText.indexOf('/'); - if (continuation >= 0) { - affixText = affixText.substring(0, continuation); + int[] continuation = new int[0]; + final int slash = affixText.indexOf('/'); + if (slash >= 0) { + continuation = parseFlags(affixText.substring(slash + 1), result.flagMode, line + 1); + affixText = affixText.substring(0, slash); } if ("0".equals(affixText)) { affixText = ""; } final Affix affix = new Affix(flag, crossProduct, strip, affixText, - AffixCondition.parse(fields[4], suffix, line + 1)); + AffixCondition.parse(fields[4], suffix, line + 1), continuation); if (suffix) { result.suffixes.add(affix); } else { diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellStemmer.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellStemmer.java index 22d32950b..ba2dd14eb 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellStemmer.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellStemmer.java @@ -100,6 +100,19 @@ public class HunspellStemmer implements Stemmer { if (flagSets != null && HunspellDictionary.hasFlag(flagSets, suffix.flag())) { analyses.add(stem); } + for (final Affix inner : dictionary.suffixes()) { + if (!inner.allowsContinuation(suffix.flag())) { + continue; + } + final String doubleStem = removeSuffix(stem, inner); + if (doubleStem == null) { + continue; + } + final List<int[]> innerFlags = dictionary.lookup(doubleStem); + if (innerFlags != null && HunspellDictionary.hasFlag(innerFlags, inner.flag())) { + analyses.add(doubleStem); + } + } } for (final Affix prefix : dictionary.prefixes()) { final String stem = removePrefix(word, prefix); diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/hunspell/HunspellStemmerTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/hunspell/HunspellStemmerTest.java index acd256117..2045f6e64 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/hunspell/HunspellStemmerTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/hunspell/HunspellStemmerTest.java @@ -117,6 +117,27 @@ public class HunspellStemmerTest { Assertions.assertEquals("lock", stemmer.stemAll("lock").get(0).toString()); } + @Test + void testTwofoldSuffixesThroughContinuationClasses() throws IOException { + final String affix = String.join("\n", + "SET UTF-8", + "SFX A Y 1", + "SFX A 0 er/B .", + "SFX B Y 1", + "SFX B 0 s .", + ""); + final String words = String.join("\n", "1", "kind/A", ""); + final HunspellDictionary dictionary = HunspellDictionary.load( + new ByteArrayInputStream(affix.getBytes(StandardCharsets.UTF_8)), + new ByteArrayInputStream(words.getBytes(StandardCharsets.UTF_8))); + final HunspellStemmer twofold = new HunspellStemmer(dictionary); + + Assertions.assertEquals("kind", twofold.stem("kinder").toString()); + Assertions.assertEquals("kind", twofold.stem("kinders").toString()); + // B alone never applies: no entry carries it directly + Assertions.assertEquals("kinds", twofold.stem("kinds").toString()); + } + @Test void testNumericFlagMode() throws IOException { final String affix = String.join("\n",
