This is an automated email from the ASF dual-hosted git repository. krickert pushed a commit to branch OPENNLP-1893-hunspell in repository https://gitbox.apache.org/repos/asf/opennlp.git
commit 50b148c2dc0e02e24a2ad3c259fb448583f5e56f Author: Kristian Rickert <[email protected]> AuthorDate: Fri Jul 17 07:03:53 2026 -0400 OPENNLP-1893: Walk only the affix rules that can apply, bucketed by their boundary character Undoing a suffix requires the word to end with the rule's affix material, so only rules whose material ends in the word's last character can ever apply, and likewise for prefixes and the first character. The dictionary now buckets its rules by that boundary character at load, and every scan in the stem path, including the twofold and cross-product inner scans, walks the one bucket plus the strip-only rules instead of the whole inventory. Measured on the LibreOffice dictionaries at 4,000 words each: English 553k to 1,024k words per second, Spanish 9.6k to 28.9k, German 132k to 287k. --- .../tools/stemmer/hunspell/HunspellDictionary.java | 66 +++++++++ .../tools/stemmer/hunspell/HunspellStemmer.java | 164 +++++++++++++++------ 2 files changed, 181 insertions(+), 49 deletions(-) diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellDictionary.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellDictionary.java index 75d47140d..c9e8f8a6e 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellDictionary.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellDictionary.java @@ -85,12 +85,44 @@ public final class HunspellDictionary { private final Map<String, List<int[]>> entries; private final List<Affix> prefixes; private final List<Affix> suffixes; + private final Map<Character, List<Affix>> suffixesByLast; + private final List<Affix> suffixesWithoutMaterial; + private final Map<Character, List<Affix>> prefixesByFirst; + private final List<Affix> prefixesWithoutMaterial; private HunspellDictionary(Map<String, List<int[]>> entries, List<Affix> prefixes, List<Affix> suffixes) { this.entries = entries; this.prefixes = prefixes; this.suffixes = suffixes; + // Undoing a suffix requires the word to end with the rule's affix material, so + // only rules whose material ends in the word's last character can ever apply; + // the same holds for prefixes and the first character. Bucketing by that + // boundary character turns the per-word rule scan from the whole inventory into + // the one bucket plus the strip-only rules, whose empty material matches + // everywhere. + this.suffixesByLast = new HashMap<>(); + this.suffixesWithoutMaterial = new ArrayList<>(); + for (final Affix suffix : suffixes) { + final String material = suffix.affix(); + if (material.isEmpty()) { + suffixesWithoutMaterial.add(suffix); + } else { + suffixesByLast.computeIfAbsent(material.charAt(material.length() - 1), + key -> new ArrayList<>()).add(suffix); + } + } + this.prefixesByFirst = new HashMap<>(); + this.prefixesWithoutMaterial = new ArrayList<>(); + for (final Affix prefix : prefixes) { + final String material = prefix.affix(); + if (material.isEmpty()) { + prefixesWithoutMaterial.add(prefix); + } else { + prefixesByFirst.computeIfAbsent(material.charAt(0), + key -> new ArrayList<>()).add(prefix); + } + } } /** @@ -159,6 +191,40 @@ public final class HunspellDictionary { return suffixes; } + private static final List<Affix> NO_AFFIXES = List.of(); + + /** + * The suffix rules whose affix material ends in the given character, which are the + * only material-bearing rules that can be undone from a word ending in it. + * + * @param last The word's last character. + * @return The bucket, possibly empty. Never {@code null}. + */ + List<Affix> suffixesEndingWith(char last) { + return suffixesByLast.getOrDefault(last, NO_AFFIXES); + } + + /** @return The strip-only suffix rules, applicable to any word. Never {@code null}. */ + List<Affix> suffixesWithoutMaterial() { + return suffixesWithoutMaterial; + } + + /** + * The prefix rules whose affix material starts with the given character, which are + * the only material-bearing rules that can be undone from a word starting with it. + * + * @param first The word's first character. + * @return The bucket, possibly empty. Never {@code null}. + */ + List<Affix> prefixesStartingWith(char first) { + return prefixesByFirst.getOrDefault(first, NO_AFFIXES); + } + + /** @return The strip-only prefix rules, applicable to any word. Never {@code null}. */ + List<Affix> prefixesWithoutMaterial() { + return prefixesWithoutMaterial; + } + /** * Checks whether any of a word's flag sets carries a flag. * diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellStemmer.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellStemmer.java index e933f4488..6e0d369c0 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellStemmer.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellStemmer.java @@ -114,55 +114,121 @@ public class HunspellStemmer implements Stemmer { if (dictionary.lookup(word) != null) { analyses.add(word); } - for (final Affix suffix : dictionary.suffixes()) { - final String stem = removeSuffix(word, suffix); - if (stem == null) { - continue; - } - final List<int[]> flagSets = dictionary.lookup(stem); - if (flagSets != null && HunspellDictionary.hasFlag(flagSets, suffix.flag())) { - analyses.add(stem); - } - for (final Affix inner : dictionary.suffixes()) { - if (!inner.allowsContinuation(suffix.flag())) { - continue; - } - final String doubleStem = removeSuffix(stem, inner); - if (doubleStem == null) { - continue; - } - final List<int[]> innerFlags = dictionary.lookup(doubleStem); - if (innerFlags != null && HunspellDictionary.hasFlag(innerFlags, inner.flag())) { - analyses.add(doubleStem); - } - } - } - for (final Affix prefix : dictionary.prefixes()) { - final String stem = removePrefix(word, prefix); - if (stem == null) { - continue; - } - final List<int[]> flagSets = dictionary.lookup(stem); - if (flagSets != null && HunspellDictionary.hasFlag(flagSets, prefix.flag())) { - analyses.add(stem); - } - if (!prefix.crossProduct()) { - continue; - } - for (final Affix suffix : dictionary.suffixes()) { - if (!suffix.crossProduct()) { - continue; - } - final String doubleStem = removeSuffix(stem, suffix); - if (doubleStem == null) { - continue; - } - final List<int[]> both = dictionary.lookup(doubleStem); - if (both != null && HunspellDictionary.hasFlag(both, prefix.flag()) - && HunspellDictionary.hasFlag(both, suffix.flag())) { - analyses.add(doubleStem); - } - } + // Only rules whose affix material ends in the word's last character can be + // undone from it, so each scan walks that bucket plus the strip-only rules + // instead of the whole inventory. + for (final Affix suffix : dictionary.suffixesEndingWith(word.charAt(word.length() - 1))) { + undoSuffix(word, suffix, analyses); + } + for (final Affix suffix : dictionary.suffixesWithoutMaterial()) { + undoSuffix(word, suffix, analyses); + } + for (final Affix prefix : dictionary.prefixesStartingWith(word.charAt(0))) { + undoPrefix(word, prefix, analyses); + } + for (final Affix prefix : dictionary.prefixesWithoutMaterial()) { + undoPrefix(word, prefix, analyses); + } + } + + /** + * Undoes one suffix rule and, through continuation classes, one further suffix on + * the intermediate stem, adding every dictionary-confirmed analysis. + * + * @param word The case variant under analysis. + * @param suffix The suffix rule to undo. + * @param analyses The mutable, insertion-ordered set collecting the stems found. + */ + private void undoSuffix(String word, Affix suffix, Set<String> analyses) { + final String stem = removeSuffix(word, suffix); + if (stem == null) { + return; + } + final List<int[]> flagSets = dictionary.lookup(stem); + if (flagSets != null && HunspellDictionary.hasFlag(flagSets, suffix.flag())) { + analyses.add(stem); + } + for (final Affix inner : dictionary.suffixesEndingWith(stem.charAt(stem.length() - 1))) { + undoInnerSuffix(stem, suffix, inner, analyses); + } + for (final Affix inner : dictionary.suffixesWithoutMaterial()) { + undoInnerSuffix(stem, suffix, inner, analyses); + } + } + + /** + * Undoes the second suffix of a twofold removal when the inner rule's continuation + * classes allow it after the outer one. + * + * @param stem The intermediate stem after the outer removal. + * @param outer The already-undone outer suffix rule. + * @param inner The candidate inner suffix rule. + * @param analyses The mutable, insertion-ordered set collecting the stems found. + */ + private void undoInnerSuffix(String stem, Affix outer, Affix inner, + Set<String> analyses) { + if (!inner.allowsContinuation(outer.flag())) { + return; + } + final String doubleStem = removeSuffix(stem, inner); + if (doubleStem == null) { + return; + } + final List<int[]> innerFlags = dictionary.lookup(doubleStem); + if (innerFlags != null && HunspellDictionary.hasFlag(innerFlags, inner.flag())) { + analyses.add(doubleStem); + } + } + + /** + * Undoes one prefix rule and, for cross-product rules, one further suffix on the + * intermediate stem, adding every dictionary-confirmed analysis. + * + * @param word The case variant under analysis. + * @param prefix The prefix rule to undo. + * @param analyses The mutable, insertion-ordered set collecting the stems found. + */ + private void undoPrefix(String word, Affix prefix, Set<String> analyses) { + final String stem = removePrefix(word, prefix); + if (stem == null) { + return; + } + final List<int[]> flagSets = dictionary.lookup(stem); + if (flagSets != null && HunspellDictionary.hasFlag(flagSets, prefix.flag())) { + analyses.add(stem); + } + if (!prefix.crossProduct()) { + return; + } + for (final Affix suffix : dictionary.suffixesEndingWith(stem.charAt(stem.length() - 1))) { + undoCrossProductSuffix(stem, prefix, suffix, analyses); + } + for (final Affix suffix : dictionary.suffixesWithoutMaterial()) { + undoCrossProductSuffix(stem, prefix, suffix, analyses); + } + } + + /** + * Undoes the suffix half of a cross-product removal when both rules opted in. + * + * @param stem The intermediate stem after the prefix removal. + * @param prefix The already-undone prefix rule. + * @param suffix The candidate suffix rule. + * @param analyses The mutable, insertion-ordered set collecting the stems found. + */ + private void undoCrossProductSuffix(String stem, Affix prefix, Affix suffix, + Set<String> analyses) { + if (!suffix.crossProduct()) { + return; + } + final String doubleStem = removeSuffix(stem, suffix); + if (doubleStem == null) { + return; + } + final List<int[]> both = dictionary.lookup(doubleStem); + if (both != null && HunspellDictionary.hasFlag(both, prefix.flag()) + && HunspellDictionary.hasFlag(both, suffix.flag())) { + analyses.add(doubleStem); } }
