This is an automated email from the ASF dual-hosted git repository. krickert pushed a commit to branch OPENNLP-1893-hunspell in repository https://gitbox.apache.org/repos/asf/opennlp.git
commit 7175791b6529ebece4097b439c06725085737896 Author: Kristian Rickert <[email protected]> AuthorDate: Fri Jul 17 06:59:10 2026 -0400 OPENNLP-1893: Resolve numeric dictionary flags through the AF alias table The published Hungarian dictionary flags all of its entries as numeric references into an AF alias table, so without alias support every entry loaded flagless and stemming answered the surface form unchanged. The affix parser now reads the AF table, the first line as the declared count and every further line as one flag run with trailing comments discarded, and a purely numeric flag field in the word list resolves as a 1-based reference into it, failing loud with the line and table size when the reference is out of range. Without an AF table numeric fields keep their FLAG num meaning. The Hungarian dictionary of the LibreOffice collection now stems inflected forms; remaining gaps there are compound territory, which is tracked separately. --- .../tools/stemmer/hunspell/HunspellDictionary.java | 47 ++++++++++++++---- .../stemmer/hunspell/HunspellStemmerTest.java | 57 ++++++++++++++++++++++ 2 files changed, 95 insertions(+), 9 deletions(-) diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellDictionary.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellDictionary.java index 14177461a..75d47140d 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellDictionary.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellDictionary.java @@ -132,8 +132,9 @@ public final class HunspellDictionary { final byte[] affixBytes = readAll(affixStream); final Charset charset = declaredCharset(affixBytes); final AffixFile affix = parseAffix(new String(affixBytes, charset)); - final Map<String, List<int[]>> entries = - parseWordList(new String(readAll(dictionaryStream), charset), affix.flagMode); + final Map<String, List<int[]>> entries = parseWordList( + new String(readAll(dictionaryStream), charset), affix.flagMode, + affix.flagAliases); return new HunspellDictionary(entries, List.copyOf(affix.prefixes), List.copyOf(affix.suffixes)); } @@ -236,14 +237,17 @@ public final class HunspellDictionary { private static final class AffixFile { private final List<Affix> prefixes = new ArrayList<>(); private final List<Affix> suffixes = new ArrayList<>(); + private final List<int[]> flagAliases = new ArrayList<>(); + private boolean aliasHeaderSeen; private FlagMode flagMode = FlagMode.CHAR; } /** - * Parses the affix file: the {@code FLAG} declaration and the {@code PFX} and - * {@code SFX} blocks. Directives outside the supported set (compounding, - * conversion tables, suggestion options, ...) are skipped, so their rules never - * fire and unsupported analyses are missed rather than invented. + * Parses the affix file: the {@code FLAG} declaration, the {@code AF} flag alias + * table, and the {@code PFX} and {@code SFX} blocks. Directives outside the + * supported set (compounding, conversion tables, suggestion options, ...) are + * skipped, so their rules never fire and unsupported analyses are missed rather + * than invented. * * @param content The decoded affix file content. * @return The parsed rules and flag mode. Never {@code null}. @@ -273,6 +277,18 @@ public final class HunspellDictionary { }; i++; break; + case "AF": + // the first AF line declares the alias count; every further AF line is one + // alias, a flag run whose 1-based position numeric dictionary flags refer to + if (fields.length >= 2) { + if (!result.aliasHeaderSeen) { + result.aliasHeaderSeen = true; + } else { + result.flagAliases.add(parseFlags(fields[1], result.flagMode, i + 1)); + } + } + i++; + break; case "PFX": case "SFX": i = parseAffixBlock(lines, i, fields, result); @@ -352,11 +368,15 @@ public final class HunspellDictionary { * * @param content The decoded word-list content. * @param flagMode The flag encoding declared by the affix file. + * @param flagAliases The affix file's {@code AF} alias table, possibly empty. When + * it is not empty, a purely numeric flag field is a 1-based + * reference into it rather than a flag run of its own. * @return The words mapped to the flag sets of their entries. Never {@code null}. - * @throws IOException Thrown if a flag run is malformed. + * @throws IOException Thrown if a flag run is malformed or an alias reference is + * out of range. */ private static Map<String, List<int[]>> parseWordList(String content, - FlagMode flagMode) throws IOException { + FlagMode flagMode, List<int[]> flagAliases) throws IOException { final String[] lines = splitLines(content); final Map<String, List<int[]>> entries = new HashMap<>(); int start = 0; @@ -385,7 +405,16 @@ public final class HunspellDictionary { break; } } - flags = parseFlags(flagRun, flagMode, i + 1); + if (!flagAliases.isEmpty() && isCount(flagRun)) { + final int alias = Integer.parseInt(flagRun); + if (alias < 1 || alias > flagAliases.size()) { + throw new IOException("flag alias " + alias + " at line " + (i + 1) + + " is outside the AF table of " + flagAliases.size() + " aliases"); + } + flags = flagAliases.get(alias - 1); + } else { + flags = parseFlags(flagRun, flagMode, i + 1); + } } entries.computeIfAbsent(word.replace("\\/", "/"), key -> new ArrayList<>(1)) .add(flags); diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/hunspell/HunspellStemmerTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/hunspell/HunspellStemmerTest.java index b116ec607..66483c701 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/hunspell/HunspellStemmerTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/hunspell/HunspellStemmerTest.java @@ -668,4 +668,61 @@ public class HunspellStemmerTest { "1\nwalk/X\n")); Assertions.assertEquals(List.of("walk"), identity.stemAll("walk")); } + + /** + * Verifies the AF flag alias table: the first AF line declares the count, every + * further AF line is one flag run, and a purely numeric flag field in the word + * list is a 1-based reference into that table, the layout the published Hungarian + * dictionary uses for all of its ninety-seven thousand entries. Alias lines may + * carry trailing comments, which the field split already discards. + * + * @throws IOException Thrown if a fixture fails to load. + */ + @Test + void testNumericDictionaryFlagsResolveThroughTheAliasTable() throws IOException { + final HunspellStemmer stemmer = new HunspellStemmer(load( + String.join("\n", + "AF 2", + "AF S # 1", + "AF SP # 2", + "SFX S Y 1", + "SFX S 0 s .", + "PFX P Y 1", + "PFX P 0 re .", + ""), + "2\nwalk/1\nplay/2\n")); + Assertions.assertEquals("walk", stemmer.stem("walks").toString()); + Assertions.assertEquals("play", stemmer.stem("plays").toString()); + Assertions.assertEquals("play", stemmer.stem("replay").toString()); + // walk carries alias 1, the suffix-only run, so the prefix must not apply + Assertions.assertEquals("rewalk", stemmer.stem("rewalk").toString()); + } + + /** + * Verifies that an alias reference outside the AF table fails loud with the line + * and the table size, instead of silently flagging the entry with nothing. + * + * @throws IOException Thrown if the affix fixture fails to load. + */ + @Test + void testAliasReferenceOutsideTheTableFailsLoud() { + final IOException e = Assertions.assertThrows(IOException.class, () -> load( + "AF 1\nAF S # 1\nSFX S Y 1\nSFX S 0 s .\n", + "1\nwalk/2\n")); + Assertions.assertEquals( + "flag alias 2 at line 2 is outside the AF table of 1 aliases", + e.getMessage()); + } + + /** + * Verifies that numeric flag fields stay ordinary flags when no AF table exists: + * under FLAG num a digit run is a flag value, not an alias reference. + * + * @throws IOException Thrown if a fixture fails to load. + */ + @Test + void testNumericFlagsWithoutAliasTableStayFlags() throws IOException { + final HunspellDictionary numbers = load("FLAG num\n", "1\nwalk/39\n"); + Assertions.assertTrue(HunspellDictionary.hasFlag(numbers.lookup("walk"), 39)); + } }
