This is an automated email from the ASF dual-hosted git repository. krickert pushed a commit to branch OPENNLP-1893-hunspell in repository https://gitbox.apache.org/repos/asf/opennlp.git
commit 24930835e5dfefd2b5c57de7327fc5e991de3c67 Author: Kristian Rickert <[email protected]> AuthorDate: Wed Jul 15 13:05:37 2026 -0400 OPENNLP-1893: Hunspell-format affix engine over user-supplied dictionaries A clean-room reader for .dic and .aff files with PFX/SFX rules, strip strings, character-class conditions matched by a single scan, cross products, and char, long, and num flag modes. No dictionary data is bundled: users point at their own files, so dictionary licenses never attach to the jar. Unsupported affix features fail closed, missing analyses rather than inventing them. (cherry picked from commit 0ecc39c7febdeaaafe1cfbdf052c9dd06ffbe4d4) --- .../tools/stemmer/hunspell/AffixCondition.java | 130 +++++++ .../tools/stemmer/hunspell/HunspellDictionary.java | 430 +++++++++++++++++++++ .../tools/stemmer/hunspell/HunspellStemmer.java | 165 ++++++++ .../stemmer/hunspell/HunspellStemmerFactory.java | 52 +++ .../stemmer/hunspell/HunspellStemmerTest.java | 181 +++++++++ 5 files changed, 958 insertions(+) diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/AffixCondition.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/AffixCondition.java new file mode 100644 index 000000000..8da9a3496 --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/AffixCondition.java @@ -0,0 +1,130 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.stemmer.hunspell; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +/** + * One parsed affix condition: a fixed-length sequence of literal characters and + * bracketed character classes, matched with a single scan and no regular expressions. + * A suffix condition anchors at the end of the candidate stem, a prefix condition at + * its start; the condition {@code .} matches everything. + */ +final class AffixCondition { + + private static final AffixCondition ANY = new AffixCondition(new char[0][], null, true); + + /** Per position: the accepted characters, or {@code null} for any character. */ + private final char[][] accepted; + /** Per position with a class: whether the class is negated; {@code null} rows unused. */ + private final boolean[] negated; + private final boolean suffix; + + private AffixCondition(char[][] accepted, boolean[] negated, boolean suffix) { + this.accepted = accepted; + this.negated = negated; + this.suffix = suffix; + } + + /** + * Parses a condition field. + * + * @param pattern The condition text from the affix rule. + * @param suffix Whether the owning rule is a suffix rule. + * @param lineNumber The affix file line, for error messages. + * @return The parsed condition. Never {@code null}. + * @throws IOException Thrown if a character class is unterminated. + */ + static AffixCondition parse(String pattern, boolean suffix, int lineNumber) + throws IOException { + if (".".equals(pattern)) { + return ANY; + } + final List<char[]> positions = new ArrayList<>(); + final List<Boolean> negations = new ArrayList<>(); + int i = 0; + while (i < pattern.length()) { + final char c = pattern.charAt(i); + if (c == '[') { + final int end = pattern.indexOf(']', i + 1); + if (end < 0) { + throw new IOException("unterminated character class at line " + lineNumber); + } + String members = pattern.substring(i + 1, end); + boolean negate = false; + if (members.startsWith("^")) { + negate = true; + members = members.substring(1); + } + positions.add(members.toCharArray()); + negations.add(negate); + i = end + 1; + } else if (c == '.') { + positions.add(null); + negations.add(false); + i++; + } else { + positions.add(new char[] {c}); + negations.add(false); + i++; + } + } + final char[][] accepted = positions.toArray(new char[0][]); + final boolean[] negated = new boolean[accepted.length]; + for (int p = 0; p < negated.length; p++) { + negated[p] = negations.get(p); + } + return new AffixCondition(accepted, negated, suffix); + } + + /** + * Tests a candidate stem against the condition at its anchored side. + * + * @param stem The candidate stem after affix removal and strip restoration. + * @return {@code true} if the stem satisfies the condition. + */ + boolean matches(String stem) { + if (accepted.length == 0) { + return true; + } + if (stem.length() < accepted.length) { + return false; + } + final int offset = suffix ? stem.length() - accepted.length : 0; + for (int p = 0; p < accepted.length; p++) { + final char[] members = accepted[p]; + if (members == null) { + continue; + } + final char c = stem.charAt(offset + p); + boolean member = false; + for (final char candidate : members) { + if (candidate == c) { + member = true; + break; + } + } + if (member == negated[p]) { + return false; + } + } + return true; + } +} diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellDictionary.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellDictionary.java new file mode 100644 index 000000000..8ac526fc3 --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellDictionary.java @@ -0,0 +1,430 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.stemmer.hunspell; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import opennlp.tools.util.StringUtil; + +/** + * An immutable, in-memory Hunspell-format dictionary: the word list of a {@code .dic} + * file and the prefix and suffix rules of its {@code .aff} companion, loaded from + * user-supplied files. The engine is a clean-room implementation of the documented + * format; no dictionary data is bundled, so the dictionaries' own licenses never attach + * to this library. + * + * <p>Supported affix features: {@code PFX} and {@code SFX} rules with strip strings, + * character-class conditions, and cross-product combination of one prefix with one + * suffix; {@code FLAG} modes {@code char} (default), {@code long}, and {@code num}; + * the {@code SET} encoding declaration. Compounding, continuation classes, and + * conversion tables are not interpreted in this version; rules using them simply do + * not fire, so unsupported analyses are missed rather than invented.</p> + * + * <p>Instances are immutable and safe to share between threads.</p> + * + * @see HunspellStemmer + * @see HunspellStemmerFactory + * @since 3.0.0 + */ +public final class HunspellDictionary { + + /** One parsed affix rule; {@code affix} is the surface material added to the stem. */ + record Affix(int flag, boolean crossProduct, String strip, String affix, + AffixCondition condition) { + } + + private final Map<String, List<int[]>> entries; + private final List<Affix> prefixes; + private final List<Affix> suffixes; + + private HunspellDictionary(Map<String, List<int[]>> entries, List<Affix> prefixes, + List<Affix> suffixes) { + this.entries = entries; + this.prefixes = prefixes; + this.suffixes = suffixes; + } + + /** + * Loads a dictionary from its two files. + * + * @param affixFile The {@code .aff} affix file. Must not be {@code null}. + * @param dictionaryFile The {@code .dic} word list. Must not be {@code null}. + * @return The loaded dictionary. Never {@code null}. + * @throws IOException Thrown if reading fails or a file is malformed. + * @throws IllegalArgumentException Thrown if a parameter is {@code null}. + */ + public static HunspellDictionary load(Path affixFile, Path dictionaryFile) + throws IOException { + if (affixFile == null || dictionaryFile == null) { + throw new IllegalArgumentException("affixFile and dictionaryFile must not be null"); + } + try (InputStream affix = Files.newInputStream(affixFile); + InputStream dictionary = Files.newInputStream(dictionaryFile)) { + return load(affix, dictionary); + } + } + + /** + * Loads a dictionary from its two streams. + * + * @param affixStream The {@code .aff} affix content. Must not be {@code null}. Not + * closed. + * @param dictionaryStream The {@code .dic} word list content. Must not be + * {@code null}. Not closed. + * @return The loaded dictionary. Never {@code null}. + * @throws IOException Thrown if reading fails or the content is malformed. + * @throws IllegalArgumentException Thrown if a parameter is {@code null}. + */ + public static HunspellDictionary load(InputStream affixStream, + InputStream dictionaryStream) throws IOException { + if (affixStream == null || dictionaryStream == null) { + throw new IllegalArgumentException("streams must not be null"); + } + final byte[] affixBytes = readAll(affixStream); + final Charset charset = declaredCharset(affixBytes); + final AffixFile affix = parseAffix(new String(affixBytes, charset)); + final Map<String, List<int[]>> entries = + parseWordList(new String(readAll(dictionaryStream), charset), affix.flagMode); + return new HunspellDictionary(entries, List.copyOf(affix.prefixes), + List.copyOf(affix.suffixes)); + } + + /** + * Looks up a word's flag sets. + * + * @param word The word exactly as listed. + * @return The flag sets of all matching entries, or {@code null} when absent. + */ + List<int[]> lookup(String word) { + return entries.get(word); + } + + /** @return The prefix rules. */ + List<Affix> prefixes() { + return prefixes; + } + + /** @return The suffix rules. */ + List<Affix> suffixes() { + return suffixes; + } + + /** + * Checks whether any of a word's flag sets carries a flag. + * + * @param flagSets The flag sets from {@link #lookup(String)}. + * @param flag The flag to look for. + * @return {@code true} if some flag set contains the flag. + */ + static boolean hasFlag(List<int[]> flagSets, int flag) { + for (final int[] flags : flagSets) { + for (final int candidate : flags) { + if (candidate == flag) { + return true; + } + } + } + return false; + } + + private static byte[] readAll(InputStream in) throws IOException { + final ByteArrayOutputStream out = new ByteArrayOutputStream(); + final byte[] buffer = new byte[8192]; + int read; + while ((read = in.read(buffer)) >= 0) { + out.write(buffer, 0, read); + } + return out.toByteArray(); + } + + /** Finds the {@code SET} declaration by scanning the raw bytes as ASCII. */ + private static Charset declaredCharset(byte[] affixBytes) throws IOException { + final String ascii = new String(affixBytes, StandardCharsets.US_ASCII); + for (final String line : splitLines(ascii)) { + final String trimmed = line.trim(); + if (trimmed.startsWith("SET ") || trimmed.startsWith("SET\t")) { + final String name = trimmed.substring(4).trim(); + try { + return Charset.forName(name); + } catch (RuntimeException e) { + throw new IOException("unsupported SET encoding: " + name, e); + } + } + } + return StandardCharsets.UTF_8; + } + + /** The flag encodings a dictionary may declare. */ + private enum FlagMode { + CHAR, LONG, NUM + } + + /** The parsed affix file content. */ + private static final class AffixFile { + private final List<Affix> prefixes = new ArrayList<>(); + private final List<Affix> suffixes = new ArrayList<>(); + private FlagMode flagMode = FlagMode.CHAR; + } + + private static AffixFile parseAffix(String content) throws IOException { + final AffixFile result = new AffixFile(); + final String[] lines = splitLines(content); + int i = 0; + while (i < lines.length) { + final String[] fields = split(lines[i]); + if (fields.length == 0 || fields[0].startsWith("#")) { + i++; + continue; + } + switch (fields[0]) { + case "FLAG": + if (fields.length < 2) { + throw new IOException("FLAG line without a mode at line " + (i + 1)); + } + result.flagMode = switch (fields[1]) { + case "long" -> FlagMode.LONG; + case "num" -> FlagMode.NUM; + default -> throw new IOException( + "unsupported FLAG mode '" + fields[1] + "' at line " + (i + 1)); + }; + i++; + break; + case "PFX": + case "SFX": + i = parseAffixBlock(lines, i, fields, result); + break; + default: + i++; + break; + } + } + return result; + } + + /** Parses one PFX or SFX header and its rule lines; returns the next line index. */ + private static int parseAffixBlock(String[] lines, int index, String[] header, + AffixFile result) throws IOException { + if (header.length < 4) { + throw new IOException("malformed affix header at line " + (index + 1)); + } + final boolean suffix = "SFX".equals(header[0]); + final int flag = parseFlag(header[1], result.flagMode, index + 1); + final boolean crossProduct = "Y".equals(header[2]); + final int count; + try { + count = Integer.parseInt(header[3]); + } catch (NumberFormatException e) { + throw new IOException("malformed affix rule count at line " + (index + 1), e); + } + int line = index + 1; + for (int rule = 0; rule < count; rule++, line++) { + if (line >= lines.length) { + throw new IOException("affix block truncated at line " + (line + 1)); + } + final String[] fields = split(lines[line]); + if (fields.length < 5 || !fields[0].equals(header[0])) { + throw new IOException("malformed affix rule at line " + (line + 1)); + } + final String strip = "0".equals(fields[2]) ? "" : fields[2]; + String affixText = fields[3]; + final int continuation = affixText.indexOf('/'); + if (continuation >= 0) { + affixText = affixText.substring(0, continuation); + } + if ("0".equals(affixText)) { + affixText = ""; + } + final Affix affix = new Affix(flag, crossProduct, strip, affixText, + AffixCondition.parse(fields[4], suffix, line + 1)); + if (suffix) { + result.suffixes.add(affix); + } else { + result.prefixes.add(affix); + } + } + return line; + } + + private static Map<String, List<int[]>> parseWordList(String content, + FlagMode flagMode) throws IOException { + final String[] lines = splitLines(content); + final Map<String, List<int[]>> entries = new HashMap<>(); + int start = 0; + if (lines.length > 0 && isCount(lines[0].trim())) { + start = 1; + } + for (int i = start; i < lines.length; i++) { + final String line = lines[i].trim(); + if (line.isEmpty()) { + continue; + } + String word = line; + int[] flags = new int[0]; + final int slash = unescapedSlash(line); + if (slash >= 0) { + word = line.substring(0, slash); + String flagText = line.substring(slash + 1); + final int fieldEnd = whitespaceIndex(flagText); + if (fieldEnd >= 0) { + flagText = flagText.substring(0, fieldEnd); + } + flags = parseFlags(flagText, flagMode, i + 1); + } else { + final int fieldEnd = whitespaceIndex(word); + if (fieldEnd >= 0) { + word = word.substring(0, fieldEnd); + } + } + entries.computeIfAbsent(word.replace("\\/", "/"), key -> new ArrayList<>(1)) + .add(flags); + } + return entries; + } + + private static boolean isCount(String line) { + if (line.isEmpty()) { + return false; + } + for (int i = 0; i < line.length(); i++) { + if (line.charAt(i) < '0' || line.charAt(i) > '9') { + return false; + } + } + return true; + } + + /** Finds the first {@code /} that is not escaped as {@code \/}. */ + private static int unescapedSlash(String line) { + for (int i = 0; i < line.length(); i++) { + if (line.charAt(i) == '/' && (i == 0 || line.charAt(i - 1) != '\\')) { + return i; + } + } + return -1; + } + + private static int whitespaceIndex(String text) { + for (int i = 0; i < text.length(); i++) { + if (StringUtil.isWhitespace(text.charAt(i))) { + return i; + } + } + return -1; + } + + private static int[] parseFlags(String text, FlagMode mode, int lineNumber) + throws IOException { + switch (mode) { + case NUM: { + final String[] parts = splitOn(text, ','); + final int[] flags = new int[parts.length]; + for (int i = 0; i < parts.length; i++) { + try { + flags[i] = Integer.parseInt(parts[i].trim()); + } catch (NumberFormatException e) { + throw new IOException("malformed numeric flag at line " + lineNumber, e); + } + } + return flags; + } + case LONG: { + if (text.length() % 2 != 0) { + throw new IOException("odd long-flag run at line " + lineNumber); + } + final int[] flags = new int[text.length() / 2]; + for (int i = 0; i < flags.length; i++) { + flags[i] = (text.charAt(2 * i) << 16) | text.charAt(2 * i + 1); + } + return flags; + } + default: { + final int[] flags = new int[text.length()]; + for (int i = 0; i < flags.length; i++) { + flags[i] = text.charAt(i); + } + return flags; + } + } + } + + private static int parseFlag(String text, FlagMode mode, int lineNumber) + throws IOException { + final int[] flags = parseFlags(text, mode, lineNumber); + if (flags.length != 1) { + throw new IOException("expected exactly one flag at line " + lineNumber); + } + return flags[0]; + } + + /** Splits text into lines with a single character scan, tolerating CRLF endings. */ + private static String[] splitLines(String content) { + final List<String> lines = new ArrayList<>(); + int start = 0; + for (int i = 0; i <= content.length(); i++) { + if (i == content.length() || content.charAt(i) == '\n') { + int end = i; + if (end > start && content.charAt(end - 1) == '\r') { + end--; + } + lines.add(content.substring(start, end)); + start = i + 1; + } + } + return lines.toArray(new String[0]); + } + + /** Splits text on a separator character with a single character scan. */ + private static String[] splitOn(String text, char separator) { + final List<String> parts = new ArrayList<>(); + int start = 0; + for (int i = 0; i <= text.length(); i++) { + if (i == text.length() || text.charAt(i) == separator) { + parts.add(text.substring(start, i)); + start = i + 1; + } + } + return parts.toArray(new String[0]); + } + + /** Splits a line on whitespace with a single character scan. */ + private static String[] split(String line) { + final List<String> parts = new ArrayList<>(); + int start = -1; + for (int i = 0; i <= line.length(); i++) { + if (i == line.length() || StringUtil.isWhitespace(line.charAt(i))) { + if (start >= 0) { + parts.add(line.substring(start, i)); + start = -1; + } + } else if (start < 0) { + start = i; + } + } + return parts.toArray(new String[0]); + } +} diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellStemmer.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellStemmer.java new file mode 100644 index 000000000..22d32950b --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellStemmer.java @@ -0,0 +1,165 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.stemmer.hunspell; + +import java.util.ArrayList; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Set; + +import opennlp.tools.stemmer.Stemmer; +import opennlp.tools.stemmer.hunspell.HunspellDictionary.Affix; +import opennlp.tools.util.StringUtil; + +/** + * A dictionary-backed {@link Stemmer} over a {@link HunspellDictionary}: a surface form + * is reduced to the dictionary words it can be derived from by removing one suffix, one + * prefix, or a cross-product combination of both. + * + * <p>{@link #stem(CharSequence)} returns the first analysis, preferring the word's own + * dictionary entry; {@link #stemAll(CharSequence)} returns every distinct analysis. A + * word with no analysis is returned unchanged, so the stemmer degrades to identity on + * unknown vocabulary. Capitalized forms also try their lowercase variant.</p> + * + * <p>The stemmer reads only immutable dictionary state and is safe to share between + * threads, satisfying the single-thread confinement contract trivially.</p> + * + * @since 3.0.0 + */ +public class HunspellStemmer implements Stemmer { + + private final HunspellDictionary dictionary; + + /** + * Initializes the stemmer. + * + * @param dictionary The dictionary to analyze against. Must not be {@code null}. + * @throws IllegalArgumentException Thrown if {@code dictionary} is {@code null}. + */ + public HunspellStemmer(HunspellDictionary dictionary) { + if (dictionary == null) { + throw new IllegalArgumentException("dictionary must not be null"); + } + this.dictionary = dictionary; + } + + @Override + public CharSequence stem(CharSequence word) { + final List<CharSequence> analyses = stemAll(word); + return analyses.get(0); + } + + @Override + public List<CharSequence> stemAll(CharSequence word) { + if (word == null) { + throw new IllegalArgumentException("word must not be null"); + } + final String surface = word.toString(); + final Set<String> analyses = new LinkedHashSet<>(); + for (final String variant : variants(surface)) { + analyze(variant, analyses); + } + if (analyses.isEmpty()) { + return List.of(surface); + } + return List.copyOf(new ArrayList<CharSequence>(analyses)); + } + + /** Collects the case variants to analyze: the surface form, then its lowercase. */ + private static List<String> variants(String surface) { + final String lowered = StringUtil.toLowerCase(surface); + return lowered.equals(surface) ? List.of(surface) : List.of(surface, lowered); + } + + /** Runs direct lookup, suffix removal, prefix removal, and cross products. */ + private void analyze(String word, Set<String> analyses) { + if (dictionary.lookup(word) != null) { + analyses.add(word); + } + for (final Affix suffix : dictionary.suffixes()) { + final String stem = removeSuffix(word, suffix); + if (stem == null) { + continue; + } + final List<int[]> flagSets = dictionary.lookup(stem); + if (flagSets != null && HunspellDictionary.hasFlag(flagSets, suffix.flag())) { + analyses.add(stem); + } + } + for (final Affix prefix : dictionary.prefixes()) { + final String stem = removePrefix(word, prefix); + if (stem == null) { + continue; + } + final List<int[]> flagSets = dictionary.lookup(stem); + if (flagSets != null && HunspellDictionary.hasFlag(flagSets, prefix.flag())) { + analyses.add(stem); + } + if (!prefix.crossProduct()) { + continue; + } + for (final Affix suffix : dictionary.suffixes()) { + if (!suffix.crossProduct()) { + continue; + } + final String doubleStem = removeSuffix(stem, suffix); + if (doubleStem == null) { + continue; + } + final List<int[]> both = dictionary.lookup(doubleStem); + if (both != null && HunspellDictionary.hasFlag(both, prefix.flag()) + && HunspellDictionary.hasFlag(both, suffix.flag())) { + analyses.add(doubleStem); + } + } + } + } + + /** + * Undoes one suffix rule. + * + * @param word The surface form. + * @param suffix The rule to undo. + * @return The candidate stem, or {@code null} when the rule does not apply. + */ + private static String removeSuffix(String word, Affix suffix) { + if (suffix.affix().isEmpty() || !word.endsWith(suffix.affix()) + || word.length() - suffix.affix().length() + suffix.strip().length() == 0) { + return null; + } + final String stem = + word.substring(0, word.length() - suffix.affix().length()) + suffix.strip(); + return suffix.condition().matches(stem) ? stem : null; + } + + /** + * Undoes one prefix rule. + * + * @param word The surface form. + * @param prefix The rule to undo. + * @return The candidate stem, or {@code null} when the rule does not apply. + */ + private static String removePrefix(String word, Affix prefix) { + if (prefix.affix().isEmpty() || !word.startsWith(prefix.affix()) + || word.length() - prefix.affix().length() + prefix.strip().length() == 0) { + return null; + } + final String stem = prefix.strip() + word.substring(prefix.affix().length()); + return prefix.condition().matches(stem) ? stem : null; + } +} diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellStemmerFactory.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellStemmerFactory.java new file mode 100644 index 000000000..eb0a8d596 --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellStemmerFactory.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.stemmer.hunspell; + +import opennlp.tools.stemmer.Stemmer; +import opennlp.tools.stemmer.StemmerFactory; + +/** + * The shareable handle for Hunspell stemming: holds one immutable + * {@link HunspellDictionary} and hands out {@link HunspellStemmer} instances over it. + * + * <p>The factory is immutable and safe to share across threads.</p> + * + * @since 3.0.0 + */ +public class HunspellStemmerFactory implements StemmerFactory { + + private final HunspellDictionary dictionary; + + /** + * Initializes the factory. + * + * @param dictionary The dictionary to stem against. Must not be {@code null}. + * @throws IllegalArgumentException Thrown if {@code dictionary} is {@code null}. + */ + public HunspellStemmerFactory(HunspellDictionary dictionary) { + if (dictionary == null) { + throw new IllegalArgumentException("dictionary must not be null"); + } + this.dictionary = dictionary; + } + + @Override + public Stemmer newStemmer() { + return new HunspellStemmer(dictionary); + } +} diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/hunspell/HunspellStemmerTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/hunspell/HunspellStemmerTest.java new file mode 100644 index 000000000..acd256117 --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/hunspell/HunspellStemmerTest.java @@ -0,0 +1,181 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.stemmer.hunspell; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +import opennlp.tools.stemmer.Stemmer; + +/** + * Tests the affix engine against a project-authored miniature dictionary; no external + * dictionary data is involved. + */ +public class HunspellStemmerTest { + + private static final String AFFIX = String.join("\n", + "# project-authored test fixture", + "SET UTF-8", + "", + "PFX U Y 1", + "PFX U 0 un .", + "", + "SFX S Y 3", + "SFX S 0 s [^sxy]", + "SFX S y ies y", + "SFX S 0 es [sx]", + "", + "SFX G Y 2", + "SFX G 0 ing [^e]", + "SFX G e ing e", + ""); + + private static final String WORDS = String.join("\n", + "6", + "lock/USG", + "pony/S", + "make/G", + "cat/S", + "box/S", + "fish", + ""); + + private static HunspellStemmer stemmer; + + @BeforeAll + static void loadDictionary() throws IOException { + final HunspellDictionary dictionary = HunspellDictionary.load( + new ByteArrayInputStream(AFFIX.getBytes(StandardCharsets.UTF_8)), + new ByteArrayInputStream(WORDS.getBytes(StandardCharsets.UTF_8))); + stemmer = new HunspellStemmer(dictionary); + } + + @Test + void testSuffixRules() { + Assertions.assertEquals("cat", stemmer.stem("cats").toString()); + Assertions.assertEquals("pony", stemmer.stem("ponies").toString()); + Assertions.assertEquals("box", stemmer.stem("boxes").toString()); + Assertions.assertEquals("make", stemmer.stem("making").toString()); + Assertions.assertEquals("lock", stemmer.stem("locking").toString()); + } + + @Test + void testPrefixAndCrossProduct() { + Assertions.assertEquals("lock", stemmer.stem("unlock").toString()); + Assertions.assertEquals("lock", stemmer.stem("unlocks").toString()); + Assertions.assertEquals("lock", stemmer.stem("unlocking").toString()); + } + + @Test + void testConditionsBlockWrongAnalyses() { + // the s rule requires a stem not ending in s, x, or y + Assertions.assertEquals("boxs", stemmer.stem("boxs").toString()); + // cat carries no G flag, so no ing analysis exists + Assertions.assertEquals("cating", stemmer.stem("cating").toString()); + // fish carries no flags at all + Assertions.assertEquals("fishs", stemmer.stem("fishs").toString()); + } + + @Test + void testDirectLookupAndCase() { + Assertions.assertEquals("fish", stemmer.stem("fish").toString()); + Assertions.assertEquals("cat", stemmer.stem("Cats").toString()); + Assertions.assertEquals("lock", stemmer.stem("Unlocks").toString()); + } + + @Test + void testUnknownWordsPassThroughUnchanged() { + Assertions.assertEquals("zebras", stemmer.stem("zebras").toString()); + Assertions.assertEquals(1, stemmer.stemAll("zebras").size()); + } + + @Test + void testStemAllReportsEveryAnalysis() { + Assertions.assertEquals(1, stemmer.stemAll("unlocks").size()); + Assertions.assertEquals("lock", stemmer.stemAll("unlocks").get(0).toString()); + // the surface form itself is an entry AND an analysis target + Assertions.assertEquals("lock", stemmer.stemAll("lock").get(0).toString()); + } + + @Test + void testNumericFlagMode() throws IOException { + final String affix = String.join("\n", + "SET UTF-8", + "FLAG num", + "SFX 100 Y 1", + "SFX 100 0 s .", + ""); + final String words = String.join("\n", "1", "walk/100,7", ""); + final HunspellDictionary dictionary = HunspellDictionary.load( + new ByteArrayInputStream(affix.getBytes(StandardCharsets.UTF_8)), + new ByteArrayInputStream(words.getBytes(StandardCharsets.UTF_8))); + Assertions.assertEquals("walk", + new HunspellStemmer(dictionary).stem("walks").toString()); + } + + @Test + void testLongFlagMode() throws IOException { + final String affix = String.join("\n", + "SET UTF-8", + "FLAG long", + "SFX Aa Y 1", + "SFX Aa 0 s .", + ""); + final String words = String.join("\n", "1", "walk/AaBb", ""); + final HunspellDictionary dictionary = HunspellDictionary.load( + new ByteArrayInputStream(affix.getBytes(StandardCharsets.UTF_8)), + new ByteArrayInputStream(words.getBytes(StandardCharsets.UTF_8))); + Assertions.assertEquals("walk", + new HunspellStemmer(dictionary).stem("walks").toString()); + } + + @Test + void testFactoryHandsOutWorkingStemmers() throws IOException { + final HunspellDictionary dictionary = HunspellDictionary.load( + new ByteArrayInputStream(AFFIX.getBytes(StandardCharsets.UTF_8)), + new ByteArrayInputStream(WORDS.getBytes(StandardCharsets.UTF_8))); + final Stemmer fresh = new HunspellStemmerFactory(dictionary).newStemmer(); + Assertions.assertEquals("pony", fresh.stem("ponies").toString()); + } + + @Test + void testMalformedInputFailsLoud() { + Assertions.assertThrows(IOException.class, () -> HunspellDictionary.load( + new ByteArrayInputStream("SFX S Y 2\nSFX S 0 s .\n".getBytes(StandardCharsets.UTF_8)), + new ByteArrayInputStream("1\ncat/S\n".getBytes(StandardCharsets.UTF_8)))); + Assertions.assertThrows(IOException.class, () -> HunspellDictionary.load( + new ByteArrayInputStream("SET NO-SUCH-ENCODING\n".getBytes(StandardCharsets.UTF_8)), + new ByteArrayInputStream("0\n".getBytes(StandardCharsets.UTF_8)))); + Assertions.assertThrows(IOException.class, () -> HunspellDictionary.load( + new ByteArrayInputStream("SFX S 0 s [a\n".getBytes(StandardCharsets.UTF_8)), + new ByteArrayInputStream("0\n".getBytes(StandardCharsets.UTF_8)))); + Assertions.assertThrows(IllegalArgumentException.class, + () -> HunspellDictionary.load((java.io.InputStream) null, + (java.io.InputStream) null)); + Assertions.assertThrows(IllegalArgumentException.class, + () -> new HunspellStemmer(null)); + Assertions.assertThrows(IllegalArgumentException.class, + () -> new HunspellStemmerFactory(null)); + Assertions.assertThrows(IllegalArgumentException.class, () -> stemmer.stemAll(null)); + } +}
