This is an automated email from the ASF dual-hosted git repository. krickert pushed a commit to branch OPENNLP-1893-hunspell in repository https://gitbox.apache.org/repos/asf/opennlp.git
commit 8bee7cb5b640cc53b7ab901db418d35d8b7ef022 Author: Kristian Rickert <[email protected]> AuthorDate: Thu Jul 16 01:45:20 2026 -0400 OPENNLP-1893: Document Hunspell dictionary acquisition with a license-preserving download helper --- .../dev/README-hunspell-dictionaries.md | 53 +++++++++++++++++ .../dev/download-hunspell-dictionary.sh | 66 ++++++++++++++++++++++ 2 files changed, 119 insertions(+) diff --git a/opennlp-core/opennlp-runtime/dev/README-hunspell-dictionaries.md b/opennlp-core/opennlp-runtime/dev/README-hunspell-dictionaries.md new file mode 100644 index 000000000..98195b029 --- /dev/null +++ b/opennlp-core/opennlp-runtime/dev/README-hunspell-dictionaries.md @@ -0,0 +1,53 @@ +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +# Hunspell dictionaries for the affix stemmer + +The Hunspell stemmer (`opennlp.tools.stemmer.hunspell`) is a clean-room engine over the documented Hunspell dictionary format: a `.dic` word list plus its `.aff` affix companion, both supplied by the user. Apache OpenNLP bundles no dictionary data, so the dictionaries' own licenses never attach to the library; whichever dictionary you download, its license is stated in the readme shipped alongside it and is yours to comply with. + +## Where dictionaries come from + +The LibreOffice project maintains a large collection of Hunspell dictionaries, one directory per language, at `github.com/LibreOffice/dictionaries`. Licenses differ per dictionary, which is exactly why nothing is bundled: for example, the `en_US` dictionary derives from SCOWL and states its terms in `README_en_US.txt` in the same directory. Many other sources work too; the engine only cares that the pair follows the Hunspell format. + +The helper next to this file fetches a pair together with its readme files: + +``` +./download-hunspell-dictionary.sh en en_US /tmp/hunspell-en_US +``` + +## Loading and stemming + +```java +import java.nio.file.Path; +import opennlp.tools.stemmer.Stemmer; +import opennlp.tools.stemmer.hunspell.HunspellDictionary; +import opennlp.tools.stemmer.hunspell.HunspellStemmerFactory; + +HunspellDictionary dictionary = HunspellDictionary.load( + Path.of("/tmp/hunspell-en_US/en_US.aff"), + Path.of("/tmp/hunspell-en_US/en_US.dic")); +HunspellStemmerFactory factory = new HunspellStemmerFactory(dictionary); + +Stemmer stemmer = factory.newStemmer(); +CharSequence stem = stemmer.stem("workers"); +``` + +The dictionary is immutable and safe to share between threads; the factory hands out a fresh stemmer per call, so each thread takes its own from `newStemmer()`. A dictionary that declares a non-UTF-8 encoding through the `SET` directive in its `.aff` file is decoded accordingly; nothing needs converting beforehand. + +## What the engine supports + +Supported affix features: `PFX` and `SFX` rules with strip strings, character-class conditions, cross-product combination of one prefix with one suffix, twofold suffixes through continuation classes, `FLAG` modes `char`, `long`, and `num`, and the `SET` encoding declaration. Compounding and conversion tables are not interpreted; rules that use them simply do not fire, so unsupported analyses are missed rather than invented. A malformed `.aff` file fails loudly at load time with the offen [...] diff --git a/opennlp-core/opennlp-runtime/dev/download-hunspell-dictionary.sh b/opennlp-core/opennlp-runtime/dev/download-hunspell-dictionary.sh new file mode 100755 index 000000000..5ed95511b --- /dev/null +++ b/opennlp-core/opennlp-runtime/dev/download-hunspell-dictionary.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Fetches one Hunspell dictionary pair (.aff and .dic) plus its license/readme files +# from the LibreOffice dictionaries collection. Each dictionary carries its own +# license, stated in the readme files this script downloads alongside it, and you +# accept that license by using the dictionary. Apache OpenNLP bundles no dictionary +# data. See README-hunspell-dictionaries.md in this directory for the Java steps that +# follow. + +set -euo pipefail + +usage() { + echo "usage: $0 <collection-dir> <dictionary-name> <target-dir>" >&2 + echo "" >&2 + echo " collection-dir the language directory inside the LibreOffice dictionaries" >&2 + echo " repository, for example: en" >&2 + echo " dictionary-name the dictionary base name inside it, for example: en_US" >&2 + echo " target-dir where the .aff, .dic, and readme files are placed" >&2 + echo "" >&2 + echo "example: $0 en en_US /tmp/hunspell-en_US" >&2 + exit 2 +} + +[ $# -ne 3 ] && usage +collection="$1" +name="$2" +target="$3" + +base="https://raw.githubusercontent.com/LibreOffice/dictionaries/master/${collection}"; +mkdir -p "${target}" + +# The .aff and .dic pair is mandatory; a missing file fails the script. +for ext in aff dic; do + echo "downloading ${name}.${ext}" + curl --fail --location --silent --show-error --retry 3 \ + --output "${target}/${name}.${ext}" "${base}/${name}.${ext}" +done + +# The readme carries the dictionary's license; keep it next to the data. Different +# collections name it differently, so try the common patterns and keep what exists. +for readme in "README_${name}.txt" "README_${collection}.txt" "README.txt" "license.txt"; do + if curl --fail --location --silent --retry 3 \ + --output "${target}/${readme}" "${base}/${readme}" 2>/dev/null; then + echo "downloaded ${readme} (contains the dictionary's license; read it)" + else + rm -f "${target}/${readme}" + fi +done + +echo "" +echo "stored ${target}/${name}.aff and ${target}/${name}.dic" +echo "next: load them from Java; see README-hunspell-dictionaries.md"
