This is an automated email from the ASF dual-hosted git repository. krickert pushed a commit to branch OPENNLP-1893-hunspell in repository https://gitbox.apache.org/repos/asf/opennlp.git
commit bd222fd67abdb56142dc759e6ba808532ca708f7 Author: Kristian Rickert <[email protected]> AuthorDate: Fri Jul 17 07:23:42 2026 -0400 OPENNLP-1893: Decompose unanalyzed words into two flagged compound parts When the affix analysis finds nothing and the affix file declares compounding, a word now splits into two listed parts that the COMPOUNDFLAG or the positional COMPOUNDBEGIN and COMPOUNDEND flags allow in their positions, honoring COMPOUNDMIN, with the parts reported left to right. Affix analyses keep precedence, listed words never decompose, and unflagged parts block a split. Against the published Hungarian dictionary the unlisted kutyahaz decomposes into its two nouns while listed compounds and inflected forms keep their regular analyses. Longer chains, syllable rules, and the compound-only flags stay unimplemented and simply leave such words unanalyzed. --- .../tools/stemmer/hunspell/HunspellDictionary.java | 77 +++++++++++++++++++++- .../tools/stemmer/hunspell/HunspellStemmer.java | 34 ++++++++++ .../stemmer/hunspell/HunspellStemmerTest.java | 50 ++++++++++++++ 3 files changed, 159 insertions(+), 2 deletions(-) diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellDictionary.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellDictionary.java index c9e8f8a6e..0f51d0e24 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellDictionary.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellDictionary.java @@ -89,9 +89,18 @@ public final class HunspellDictionary { private final List<Affix> suffixesWithoutMaterial; private final Map<Character, List<Affix>> prefixesByFirst; private final List<Affix> prefixesWithoutMaterial; + private final int compoundFlag; + private final int compoundBegin; + private final int compoundEnd; + private final int compoundMin; private HunspellDictionary(Map<String, List<int[]>> entries, List<Affix> prefixes, - List<Affix> suffixes) { + List<Affix> suffixes, int compoundFlag, int compoundBegin, int compoundEnd, + int compoundMin) { + this.compoundFlag = compoundFlag; + this.compoundBegin = compoundBegin; + this.compoundEnd = compoundEnd; + this.compoundMin = compoundMin; this.entries = entries; this.prefixes = prefixes; this.suffixes = suffixes; @@ -168,7 +177,8 @@ public final class HunspellDictionary { new String(readAll(dictionaryStream), charset), affix.flagMode, affix.flagAliases); return new HunspellDictionary(entries, List.copyOf(affix.prefixes), - List.copyOf(affix.suffixes)); + List.copyOf(affix.suffixes), affix.compoundFlag, affix.compoundBegin, + affix.compoundEnd, affix.compoundMin); } /** @@ -225,6 +235,40 @@ public final class HunspellDictionary { return prefixesWithoutMaterial; } + /** @return Whether the affix file declares any compounding flag at all. */ + boolean compoundsDeclared() { + return compoundFlag != 0 || compoundBegin != 0 || compoundEnd != 0; + } + + /** @return The smallest length a compound part may have; at least {@code 1}. */ + int compoundMin() { + return compoundMin; + } + + /** + * Checks whether a listed word may open a compound: it carries the general + * compounding flag or the dedicated begin flag. + * + * @param flagSets The word's flag sets from {@link #lookup(String)}. + * @return {@code true} if the word may stand first in a compound. + */ + boolean mayBeginCompound(List<int[]> flagSets) { + return (compoundFlag != 0 && hasFlag(flagSets, compoundFlag)) + || (compoundBegin != 0 && hasFlag(flagSets, compoundBegin)); + } + + /** + * Checks whether a listed word may close a compound: it carries the general + * compounding flag or the dedicated end flag. + * + * @param flagSets The word's flag sets from {@link #lookup(String)}. + * @return {@code true} if the word may stand last in a compound. + */ + boolean mayEndCompound(List<int[]> flagSets) { + return (compoundFlag != 0 && hasFlag(flagSets, compoundFlag)) + || (compoundEnd != 0 && hasFlag(flagSets, compoundEnd)); + } + /** * Checks whether any of a word's flag sets carries a flag. * @@ -306,6 +350,10 @@ public final class HunspellDictionary { private final List<int[]> flagAliases = new ArrayList<>(); private boolean aliasHeaderSeen; private FlagMode flagMode = FlagMode.CHAR; + private int compoundFlag; + private int compoundBegin; + private int compoundEnd; + private int compoundMin = 3; } /** @@ -343,6 +391,31 @@ public final class HunspellDictionary { }; i++; break; + case "COMPOUNDFLAG": + case "COMPOUNDBEGIN": + case "COMPOUNDEND": + if (fields.length < 2) { + throw new IOException(fields[0] + " line without a flag at line " + (i + 1)); + } + final int compound = parseFlag(fields[1], result.flagMode, i + 1); + switch (fields[0]) { + case "COMPOUNDFLAG" -> result.compoundFlag = compound; + case "COMPOUNDBEGIN" -> result.compoundBegin = compound; + default -> result.compoundEnd = compound; + } + i++; + break; + case "COMPOUNDMIN": + if (fields.length < 2) { + throw new IOException("COMPOUNDMIN line without a value at line " + (i + 1)); + } + try { + result.compoundMin = Math.max(1, Integer.parseInt(fields[1])); + } catch (NumberFormatException e) { + throw new IOException("malformed COMPOUNDMIN at line " + (i + 1), e); + } + i++; + break; case "AF": // the first AF line declares the alias count; every further AF line is one // alias, a flag run whose 1-based position numeric dictionary flags refer to diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellStemmer.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellStemmer.java index 6e0d369c0..6ac1a48aa 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellStemmer.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellStemmer.java @@ -81,6 +81,11 @@ public class HunspellStemmer implements Stemmer { for (final String variant : variants(surface)) { analyze(variant, analyses); } + if (analyses.isEmpty() && dictionary.compoundsDeclared()) { + for (final String variant : variants(surface)) { + decompose(variant, analyses); + } + } if (analyses.isEmpty()) { return List.of(surface); } @@ -131,6 +136,35 @@ public class HunspellStemmer implements Stemmer { } } + /** + * Decomposes a word into two listed compound parts when the affix analysis found + * nothing: at every split point that leaves both sides at least the declared + * minimum length, the left side must be listed and allowed to open a compound and + * the right side listed and allowed to close one. The parts of the first splitting + * that succeeds are reported left to right, so the head-most material comes last, + * and further splittings add any parts not already reported. + * + * @param word The case variant to decompose. + * @param analyses The mutable, insertion-ordered set collecting the parts. + */ + private void decompose(String word, Set<String> analyses) { + final int min = dictionary.compoundMin(); + for (int split = min; split <= word.length() - min; split++) { + final String left = word.substring(0, split); + final List<int[]> leftFlags = dictionary.lookup(left); + if (leftFlags == null || !dictionary.mayBeginCompound(leftFlags)) { + continue; + } + final String right = word.substring(split); + final List<int[]> rightFlags = dictionary.lookup(right); + if (rightFlags == null || !dictionary.mayEndCompound(rightFlags)) { + continue; + } + analyses.add(left); + analyses.add(right); + } + } + /** * Undoes one suffix rule and, through continuation classes, one further suffix on * the intermediate stem, adding every dictionary-confirmed analysis. diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/hunspell/HunspellStemmerTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/hunspell/HunspellStemmerTest.java index 66483c701..9ffd1289b 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/hunspell/HunspellStemmerTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/hunspell/HunspellStemmerTest.java @@ -725,4 +725,54 @@ public class HunspellStemmerTest { final HunspellDictionary numbers = load("FLAG num\n", "1\nwalk/39\n"); Assertions.assertTrue(HunspellDictionary.hasFlag(numbers.lookup("walk"), 39)); } + + /** + * Verifies two-part compound decomposition under the general compounding flag: a + * word the affix analysis cannot explain splits into two listed parts that both + * carry the flag, reported left to right, while a part without the flag blocks the + * split and the word stays unanalyzed. + * + * @throws IOException Thrown if a fixture fails to load. + */ + @Test + void testCompoundFlagDecomposesUnanalyzedWords() throws IOException { + final HunspellStemmer stemmer = new HunspellStemmer(load( + "COMPOUNDFLAG Z\nCOMPOUNDMIN 3\n", + "3\ndog/Z\nhouse/Z\ncat\n")); + Assertions.assertEquals(List.of("dog", "house"), stemmer.stemAll("doghouse")); + // cat is listed without the compounding flag, so no split may use it + Assertions.assertEquals(List.of("cathouse"), stemmer.stemAll("cathouse")); + // a listed word never decomposes; it is its own analysis + Assertions.assertEquals(List.of("dog"), stemmer.stemAll("dog")); + } + + /** + * Verifies the positional compound flags: the begin flag only opens and the end + * flag only closes, so the parts compose in one order and refuse the other. + * + * @throws IOException Thrown if a fixture fails to load. + */ + @Test + void testCompoundBeginAndEndFlagsArePositional() throws IOException { + final HunspellStemmer stemmer = new HunspellStemmer(load( + "COMPOUNDBEGIN B\nCOMPOUNDEND E\nCOMPOUNDMIN 3\n", + "2\ndog/B\nhouse/E\n")); + Assertions.assertEquals(List.of("dog", "house"), stemmer.stemAll("doghouse")); + Assertions.assertEquals(List.of("housedog"), stemmer.stemAll("housedog")); + } + + /** + * Verifies the minimum part length: a split leaving a side shorter than + * COMPOUNDMIN is never taken, although both sides are listed and flagged. + * + * @throws IOException Thrown if a fixture fails to load. + */ + @Test + void testCompoundMinBoundsThePartLength() throws IOException { + final HunspellStemmer stemmer = new HunspellStemmer(load( + "COMPOUNDFLAG Z\nCOMPOUNDMIN 4\n", + "2\ndog/Z\nhouse/Z\n")); + // the left side would be three characters, below the declared minimum of four + Assertions.assertEquals(List.of("doghouse"), stemmer.stemAll("doghouse")); + } }
