This is an automated email from the ASF dual-hosted git repository. krickert pushed a commit to branch OPENNLP-1893-hunspell in repository https://gitbox.apache.org/repos/asf/opennlp.git
commit 3c2da75e7e7bb996bf06b9634b043ef11c12e20e Author: Kristian Rickert <[email protected]> AuthorDate: Fri Jul 17 01:08:34 2026 -0400 OPENNLP-1893: Read flags as code points, tolerate trailing morphology, and stem nothing from nothing Loading the Spanish dictionary of the LibreOffice collection, the same collection this module's README recommends, exposed three gaps against real data. Flags under FLAG UTF-8 are now one code point each instead of one UTF-16 unit, since that dictionary names prefix rules with supplementary characters that would otherwise split into two flags and abort the load; a variation selector after a flag character selects presentation, not identity, and is dropped, which the same file also relies on. A numeric or long flag run ends at the first space or tabulator, the separators the word-list format defines, so trailing morphological text without a tag no longer aborts the load; the morphology cut itself now splits on exactly those two separators, the set the reference implementation's hashmgr.cxx uses, which the javadoc previously claimed while scanning wider whitespace. Stemming the empty word answers the empty word instead of letting a strip-only rule conjure a stem from nothing. All four downloaded dictionaries of the collection, English, Spanish, Hungarian, and German, now load and stem; new tests pin the escaped slash, the multi-word entry with trailing tags, and each corrected behavior. --- .../tools/stemmer/hunspell/HunspellDictionary.java | 60 ++++++++-- .../tools/stemmer/hunspell/HunspellStemmer.java | 5 + .../stemmer/hunspell/HunspellStemmerTest.java | 129 +++++++++++++++++++++ 3 files changed, 184 insertions(+), 10 deletions(-) diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellDictionary.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellDictionary.java index 6dbb001eb..14177461a 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellDictionary.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellDictionary.java @@ -25,6 +25,7 @@ import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -374,7 +375,17 @@ public final class HunspellDictionary { final int slash = unescapedSlash(entry); if (slash >= 0) { word = entry.substring(0, slash); - flags = parseFlags(entry.substring(slash + 1), flagMode, i + 1); + String flagRun = entry.substring(slash + 1); + // The flag run ends at the first space or tabulator, the separators the + // word-list format defines; whatever follows is a morphological field even + // when it carries no two-letter tag, which hunspell tolerates and so do we. + for (int c = 0; c < flagRun.length(); c++) { + if (isFieldSeparator(flagRun.charAt(c))) { + flagRun = flagRun.substring(0, c); + break; + } + } + flags = parseFlags(flagRun, flagMode, i + 1); } entries.computeIfAbsent(word.replace("\\/", "/"), key -> new ArrayList<>(1)) .add(flags); @@ -421,9 +432,12 @@ public final class HunspellDictionary { * Finds where the trailing morphological fields of a word-list entry begin, which * terminates the word and its flag run. A morphological field is either introduced by * a tabulator, the older separator, or written as a two-letter tag followed by - * {@code :} and preceded by whitespace, such as {@code po:verb}. Whitespace that is - * not followed by such a tag belongs to the word, because a word-list entry may name - * several words. + * {@code :} and preceded by a separator, such as {@code po:verb}. A separator that + * is not followed by such a tag belongs to the word, because a word-list entry may + * name several words. The separators are the space and the tabulator, exactly the + * two characters the reference implementation's {@code hashmgr.cxx} splits on; they + * are format delimiters of the word-list grammar, not a whitespace judgment, so + * wider whitespace such as a no-break space stays part of the word by design. * * @param line The trimmed word-list line to scan. * @return The index at which the morphological fields begin, or {@code -1} if the @@ -432,9 +446,9 @@ public final class HunspellDictionary { private static int morphologyIndex(String line) { int cut = -1; for (int i = 4; i < line.length(); i++) { - if (line.charAt(i) == ':' && StringUtil.isWhitespace(line.charAt(i - 3))) { + if (line.charAt(i) == ':' && isFieldSeparator(line.charAt(i - 3))) { int fieldStart = i - 3; - while (fieldStart > 0 && StringUtil.isWhitespace(line.charAt(fieldStart - 1))) { + while (fieldStart > 0 && isFieldSeparator(line.charAt(fieldStart - 1))) { fieldStart--; } // a tag with no word in front of it is not a morphological field @@ -449,6 +463,18 @@ public final class HunspellDictionary { return cut; } + /** + * Checks one character against the word-list format's field separators, space and + * tabulator, the exact set the reference implementation splits morphological fields + * on. + * + * @param c The character to test. + * @return {@code true} if {@code c} separates fields in the word-list format. + */ + private static boolean isFieldSeparator(char c) { + return c == ' ' || c == '\t'; + } + /** * Removes leading and trailing whitespace, using the whitespace definition the rest * of the parser scans with. @@ -509,11 +535,25 @@ public final class HunspellDictionary { return flags; } default: { - final int[] flags = new int[text.length()]; - for (int i = 0; i < flags.length; i++) { - flags[i] = text.charAt(i); + // One flag per code point: published dictionaries, the Spanish one of the + // LibreOffice collection among them, name affix rules with supplementary + // characters under FLAG UTF-8, and reading per UTF-16 unit would split such + // a flag into two and reject the rule header as carrying two flags. A + // variation selector after a flag character selects its presentation, the + // emoji telephone against the text telephone, and is no flag of its own; the + // same collection writes such selectors, so they are dropped from flag + // identity. + final int[] buffer = new int[text.codePointCount(0, text.length())]; + int f = 0; + for (int i = 0; i < text.length(); ) { + final int codePoint = text.codePointAt(i); + i += Character.charCount(codePoint); + if (codePoint >= 0xFE00 && codePoint <= 0xFE0F) { + continue; + } + buffer[f++] = codePoint; } - return flags; + return f == buffer.length ? buffer : Arrays.copyOf(buffer, f); } } } diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellStemmer.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellStemmer.java index eb55bb6d8..e933f4488 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellStemmer.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellStemmer.java @@ -72,6 +72,11 @@ public class HunspellStemmer implements Stemmer { throw new IllegalArgumentException("word must not be null"); } final String surface = word.toString(); + if (surface.isEmpty()) { + // a zero-length word has no morphology; without this guard a strip-only rule + // could restore its strip string onto nothing and answer a non-empty stem + return List.of(surface); + } final Set<String> analyses = new LinkedHashSet<>(); for (final String variant : variants(surface)) { analyze(variant, analyses); diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/hunspell/HunspellStemmerTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/hunspell/HunspellStemmerTest.java index 82956bd9b..b116ec607 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/hunspell/HunspellStemmerTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/hunspell/HunspellStemmerTest.java @@ -21,6 +21,7 @@ import java.io.ByteArrayInputStream; import java.io.IOException; import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; +import java.util.List; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; @@ -539,4 +540,132 @@ public class HunspellStemmerTest { () -> new HunspellStemmerFactory(null)); Assertions.assertThrows(IllegalArgumentException.class, () -> stemmer.stemAll(null)); } + + /** + * Verifies hunspell's tolerance for trailing text after a numeric or long flag run: + * the flag run ends at the first space, and whatever follows is a morphological + * field even without a two-letter tag, so such an entry loads instead of aborting + * the whole dictionary. + * + * @throws IOException Thrown if a fixture fails to load. + */ + @Test + void testTrailingTextAfterNumericFlagRunIsMorphologyNotAnError() throws IOException { + final HunspellDictionary numbers = load("FLAG num\n", + "2\nwalk/39 blah\nrun/7,9 xyz abc\n"); + Assertions.assertNotNull(numbers.lookup("walk")); + Assertions.assertTrue(HunspellDictionary.hasFlag(numbers.lookup("walk"), 39)); + Assertions.assertTrue(HunspellDictionary.hasFlag(numbers.lookup("run"), 7)); + Assertions.assertTrue(HunspellDictionary.hasFlag(numbers.lookup("run"), 9)); + + final HunspellDictionary longs = load("FLAG long\n", "1\nwalk/AB cd\n"); + Assertions.assertTrue(HunspellDictionary.hasFlag(longs.lookup("walk"), + ('A' << 16) | 'B')); + } + + /** + * Verifies that stemming the empty word answers the empty word: a zero-length + * surface has no morphology, and a strip-only rule must not restore its strip + * string onto nothing and answer a non-empty stem. + * + * @throws IOException Thrown if a fixture fails to load. + */ + @Test + void testEmptyWordStemsToItself() throws IOException { + final HunspellStemmer stripOnly = new HunspellStemmer(load( + "PFX P Y 1\nPFX P xy 0 .\n", + "1\nxy/P\n")); + Assertions.assertEquals("", stripOnly.stem("").toString()); + Assertions.assertEquals(List.of(""), stripOnly.stemAll("")); + } + + /** + * Verifies the escaped-slash feature: {@code \/} belongs to the word, so an entry + * naming a slashed term keeps its slash while the first unescaped slash still + * separates the flag run. + * + * @throws IOException Thrown if a fixture fails to load. + */ + @Test + void testEscapedSlashBelongsToTheWord() throws IOException { + final HunspellDictionary slashed = load("FLAG num\n", + "2\nTCP\\/IP/39\nAC\\/DC\n"); + Assertions.assertNotNull(slashed.lookup("TCP/IP")); + Assertions.assertTrue(HunspellDictionary.hasFlag(slashed.lookup("TCP/IP"), 39)); + Assertions.assertNotNull(slashed.lookup("AC/DC")); + Assertions.assertNull(slashed.lookup("TCP")); + } + + /** + * Verifies the sharpest combination of the morphology cut: an entry that is both a + * multi-word term and carries trailing tag morphology keeps the whole multi-word + * surface and its flags, and the tags stay out of the word. + * + * @throws IOException Thrown if a fixture fails to load. + */ + @Test + void testMultiWordEntryWithTrailingTagMorphology() throws IOException { + final HunspellDictionary phrases = load("FLAG num\n", + "1\nall right/39 po:phrase st:allright\n"); + Assertions.assertNotNull(phrases.lookup("all right")); + Assertions.assertTrue(HunspellDictionary.hasFlag(phrases.lookup("all right"), 39)); + Assertions.assertNull(phrases.lookup("all right po:phrase st:allright")); + } + + /** + * Pins FLAG UTF-8 for a supplementary flag character: a flag is one code point, so + * a character above U+FFFF is one flag carrying its code point value, never two + * surrogate-unit flags. The Spanish dictionary of the LibreOffice collection names + * affix rules with such characters, so an affix keyed by a supplementary flag must + * connect to the entries that carry it. + * + * @throws IOException Thrown if a fixture fails to load. + */ + @Test + void testSupplementaryFlagCharacterIsOneCodePointFlag() throws IOException { + // U+1F600 as a flag, written as its surrogate pair + final HunspellDictionary emoji = load("FLAG UTF-8\n", + "1\nwalk/\uD83D\uDE00\n"); + Assertions.assertTrue(HunspellDictionary.hasFlag(emoji.lookup("walk"), 0x1F600)); + Assertions.assertFalse(HunspellDictionary.hasFlag(emoji.lookup("walk"), 0xD83D)); + + final HunspellStemmer stemmer = new HunspellStemmer(load( + "FLAG UTF-8\nSFX \uD83D\uDE00 Y 1\nSFX \uD83D\uDE00 0 s .\n", + "1\nwalk/\uD83D\uDE00\n")); + Assertions.assertEquals("walk", stemmer.stem("walks").toString()); + } + + /** + * Pins the variation-selector rule the Spanish dictionary of the LibreOffice + * collection relies on: a variation selector after a flag character selects its + * presentation and is no flag of its own, so an affix rule named with the emoji + * form of a character connects to entries flagged with either spelling. + * + * @throws IOException Thrown if a fixture fails to load. + */ + @Test + void testVariationSelectorIsDroppedFromFlagIdentity() throws IOException { + // U+260E BLACK TELEPHONE followed by U+FE0F VARIATION SELECTOR-16, the exact + // shape of a prefix flag in the published es_ES affix file + final HunspellStemmer stemmer = new HunspellStemmer(load( + "FLAG UTF-8\nPFX \u260E\uFE0F Y 1\nPFX \u260E\uFE0F 0 tele .\n", + "1\nfono/\u260E\n")); + Assertions.assertEquals("fono", stemmer.stem("telefono").toString()); + } + + /** + * Pins the documented rejection of rules that neither add nor remove material: a + * suffix rule with strip {@code 0} and affix {@code 0} loads without error and + * never fires, so stemming a flagged dictionary word answers that word exactly + * once. + * + * @throws IOException Thrown if a fixture fails to load. + */ + @Test + void testRuleThatNeitherAddsNorRemovesLoadsAndNeverFires() throws IOException { + final HunspellStemmer identity = new HunspellStemmer(load( + "SFX X Y 1\nSFX X 0 0 .\n", + "1\nwalk/X\n")); + Assertions.assertEquals(List.of("walk"), identity.stemAll("walk")); + } }
