This is an automated email from the ASF dual-hosted git repository.
mawiesne pushed a commit to branch OPENNLP-1850-2b-term
in repository https://gitbox.apache.org/repos/asf/opennlp.git
The following commit(s) were added to refs/heads/OPENNLP-1850-2b-term by this
push:
new 56c472e64 OPENNLP-1864: Per-language NormalizationProfile registry
(#1112)
56c472e64 is described below
commit 56c472e64985c7e45b3d77eb9cbdfcc9e8ece600
Author: Kristian Rickert <[email protected]>
AuthorDate: Mon Jul 6 08:37:25 2026 -0400
OPENNLP-1864: Per-language NormalizationProfile registry (#1112)
* OPENNLP-1864: Per-language NormalizationProfile registry (2c)
* For broader context, see epic:
https://issues.apache.org/jira/browse/OPENNLP-1850
---
.../util/normalizer/NormalizationProfile.java | 85 +++++++++
.../util/normalizer/NormalizationProfiles.java | 139 ++++++++++++++
.../util/normalizer/NormalizationProfilesTest.java | 204 +++++++++++++++++++++
3 files changed, 428 insertions(+)
diff --git
a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/NormalizationProfile.java
b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/NormalizationProfile.java
new file mode 100644
index 000000000..e7a47477e
--- /dev/null
+++
b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/NormalizationProfile.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.tools.util.normalizer;
+
+import opennlp.tools.stemmer.Stemmer;
+import opennlp.tools.stemmer.snowball.SnowballStemmer;
+
+/**
+ * Per-language normalization settings, mirroring how OpenNLP already selects
a Snowball stemmer by
+ * language. A profile pairs a language with its Snowball {@link
SnowballStemmer.ALGORITHM} and the
+ * diacritic fold appropriate for that language (if any).
+ *
+ * <p>The {@code accentFold} normalizer is the language's diacritic transform
for a matching form, or
+ * {@code null} when folding is not appropriate. It is the generic
+ * {@link AccentFoldCharSequenceNormalizer} for English and the major Romance
languages (where
+ * accented letters are matching variants of their base letter), the
German-specific
+ * {@link GermanUmlautCharSequenceNormalizer} (a-umlaut to {@code ae}, eszett
to {@code ss}, ...) for
+ * German, and {@code null} where diacritics mark distinct letters (the Nordic
languages and the
+ * non-Latin scripts), because folding there is language-wrong. This is a
search-recall choice, not a
+ * statement of linguistic correctness; callers can build a {@link
TermAnalyzer} directly to
+ * override it.</p>
+ *
+ * @param language The language, as an ISO 639-3 code (for example
{@code "eng"}). Must
+ * not be {@code null} or blank.
+ * @param stemmerAlgorithm The Snowball algorithm for the language. Must not
be {@code null}.
+ * @param accentFold The diacritic fold for the language, or {@code
null} for none.
+ */
+public record NormalizationProfile(String language, SnowballStemmer.ALGORITHM
stemmerAlgorithm,
+ CharSequenceNormalizer accentFold) {
+
+ /**
+ * Validates the components.
+ *
+ * @throws IllegalArgumentException if {@code language} or {@code
stemmerAlgorithm} is
+ * {@code null}, or if {@code language} is blank.
+ */
+ public NormalizationProfile {
+ if (language == null) {
+ throw new IllegalArgumentException("language must not be null");
+ }
+ if (stemmerAlgorithm == null) {
+ throw new IllegalArgumentException("stemmerAlgorithm must not be null");
+ }
+ if (language.isBlank()) {
+ throw new IllegalArgumentException("language must not be blank");
+ }
+ }
+
+ /**
+ * {@return a new {@link Stemmer} for this language} A fresh instance is
returned on each call
+ * because the Snowball stemmers are stateful and not thread-safe.
+ */
+ public Stemmer newStemmer() {
+ return new SnowballStemmer(stemmerAlgorithm);
+ }
+
+ /**
+ * Returns a matching analyzer for this language: NFC, case folding, the
language's
+ * {@linkplain #accentFold() diacritic fold} when it has one, then stemming.
Each call builds an
+ * independent analyzer with its own stemmer, so use one per thread when
stemming.
+ *
+ * @return the analyzer.
+ */
+ public TermAnalyzer matchingAnalyzer() {
+ final TermAnalyzer.Builder builder =
TermAnalyzer.builder().nfc().caseFold();
+ if (accentFold != null) {
+ builder.transform(Dimension.ACCENT_FOLD, accentFold);
+ }
+ return builder.stem(newStemmer()).build();
+ }
+}
diff --git
a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/NormalizationProfiles.java
b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/NormalizationProfiles.java
new file mode 100644
index 000000000..70b2f560c
--- /dev/null
+++
b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/NormalizationProfiles.java
@@ -0,0 +1,139 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.tools.util.normalizer;
+
+import java.util.HashMap;
+import java.util.Locale;
+import java.util.Map;
+import java.util.MissingResourceException;
+import java.util.Optional;
+import java.util.Set;
+
+import opennlp.tools.langdetect.LanguageDetector;
+import opennlp.tools.stemmer.snowball.SnowballStemmer;
+
+/**
+ * A registry of {@link NormalizationProfile}s by language, with
detection-based fallback. This is
+ * the language dispatch the design note calls for: pick the profile for a
requested language, or
+ * detect the language with a {@link LanguageDetector} when it is unspecified.
The covered languages
+ * are the Snowball stemmer algorithms that name a natural language -- every
+ * {@link SnowballStemmer.ALGORITHM} except {@code PORTER}, which is an
English-only algorithm
+ * variant rather than a distinct language. Several codes can map to one
algorithm (the three
+ * Norwegian written standards all use {@code NORWEGIAN}).
+ *
+ * <p>Profiles are keyed by ISO 639-3 code (what {@link LanguageDetector}
produces);
+ * {@link #forLanguage(String)} also accepts ISO 639-1 two-letter codes.</p>
+ */
+public final class NormalizationProfiles {
+
+ private static final Map<String, NormalizationProfile> BY_LANGUAGE = build();
+
+ private NormalizationProfiles() {
+ }
+
+ private static Map<String, NormalizationProfile> build() {
+ final Map<String, NormalizationProfile> map = new HashMap<>();
+ // The generic accent fold is used for English and the major Romance
languages, German uses its
+ // own ae/oe/ue/ss fold, and folding is disabled elsewhere (Nordic,
non-Latin) where diacritics
+ // mark distinct letters.
+ final CharSequenceNormalizer latin =
AccentFoldCharSequenceNormalizer.getInstance();
+ final CharSequenceNormalizer german =
GermanUmlautCharSequenceNormalizer.getInstance();
+ add(map, "ara", SnowballStemmer.ALGORITHM.ARABIC, null);
+ add(map, "cat", SnowballStemmer.ALGORITHM.CATALAN, latin);
+ add(map, "dan", SnowballStemmer.ALGORITHM.DANISH, null);
+ add(map, "deu", SnowballStemmer.ALGORITHM.GERMAN, german);
+ add(map, "ell", SnowballStemmer.ALGORITHM.GREEK, null);
+ add(map, "eng", SnowballStemmer.ALGORITHM.ENGLISH, latin);
+ add(map, "fin", SnowballStemmer.ALGORITHM.FINNISH, null);
+ add(map, "fra", SnowballStemmer.ALGORITHM.FRENCH, latin);
+ add(map, "gle", SnowballStemmer.ALGORITHM.IRISH, null);
+ add(map, "hun", SnowballStemmer.ALGORITHM.HUNGARIAN, null);
+ add(map, "ind", SnowballStemmer.ALGORITHM.INDONESIAN, null);
+ add(map, "ita", SnowballStemmer.ALGORITHM.ITALIAN, latin);
+ add(map, "nld", SnowballStemmer.ALGORITHM.DUTCH, null);
+ add(map, "nob", SnowballStemmer.ALGORITHM.NORWEGIAN, null); // Bokmal
(nb), the dominant standard
+ add(map, "nno", SnowballStemmer.ALGORITHM.NORWEGIAN, null); // Nynorsk (nn)
+ add(map, "nor", SnowballStemmer.ALGORITHM.NORWEGIAN, null); //
macrolanguage / 639-1 "no"
+ add(map, "por", SnowballStemmer.ALGORITHM.PORTUGUESE, latin);
+ add(map, "ron", SnowballStemmer.ALGORITHM.ROMANIAN, null);
+ add(map, "rus", SnowballStemmer.ALGORITHM.RUSSIAN, null);
+ add(map, "spa", SnowballStemmer.ALGORITHM.SPANISH, latin);
+ add(map, "swe", SnowballStemmer.ALGORITHM.SWEDISH, null);
+ // Turkish diacritics are distinct letters, so there is no accent fold.
The matching analyzer's
+ // case fold stays locale-generic: the Turkish dotted/dotless-i pair folds
by the Unicode default
+ // rather than Turkish rules -- a deliberate recall choice, not
Turkish-correct casing.
+ add(map, "tur", SnowballStemmer.ALGORITHM.TURKISH, null);
+ return Map.copyOf(map);
+ }
+
+ private static void add(Map<String, NormalizationProfile> map, String
language,
+ SnowballStemmer.ALGORITHM algorithm, CharSequenceNormalizer accentFold) {
+ map.put(language, new NormalizationProfile(language, algorithm,
accentFold));
+ }
+
+ /**
+ * Returns the {@link NormalizationProfile profile} for a language.
+ *
+ * @param language An ISO 639-3 or ISO 639-1 language code;
case-insensitive. Must not be
+ * {@code null}.
+ * @return The profile, or empty if the language has no Snowball stemmer.
+ * @throws IllegalArgumentException if {@code language} is {@code null}.
+ */
+ public static Optional<NormalizationProfile> forLanguage(String language) {
+ if (language == null) {
+ throw new IllegalArgumentException("language must not be null");
+ }
+ String code = language.strip().toLowerCase(Locale.ROOT);
+ if (code.length() == 2) {
+ try {
+ final String iso3 = Locale.of(code).getISO3Language();
+ if (!iso3.isEmpty()) {
+ code = iso3;
+ }
+ } catch (MissingResourceException ignored) {
+ // No ISO 639-3 code for this two-letter code; fall through and look
up as given.
+ }
+ }
+ return Optional.ofNullable(BY_LANGUAGE.get(code));
+ }
+
+ /**
+ * Detects the language of {@code text} and returns its {@link
NormalizationProfile profile}.
+ *
+ * @param text The text to detect. Must not be {@code null}.
+ * @param detector The language detector to use. Must not be {@code null}.
+ * @return The profile for the detected language, or empty if it has no
Snowball stemmer.
+ * @throws IllegalArgumentException if {@code text} or {@code detector} is
{@code null}.
+ */
+ public static Optional<NormalizationProfile> detect(CharSequence text,
+ LanguageDetector detector) {
+ if (text == null) {
+ throw new IllegalArgumentException("text must not be null");
+ }
+ if (detector == null) {
+ throw new IllegalArgumentException("detector must not be null");
+ }
+ return forLanguage(detector.predictLanguage(text).getLang());
+ }
+
+ /**
+ * {@return the ISO 639-3 codes of the supported languages}
+ */
+ public static Set<String> supportedLanguages() {
+ return BY_LANGUAGE.keySet();
+ }
+}
diff --git
a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/NormalizationProfilesTest.java
b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/NormalizationProfilesTest.java
new file mode 100644
index 000000000..ba39a5cf1
--- /dev/null
+++
b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/NormalizationProfilesTest.java
@@ -0,0 +1,204 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.tools.util.normalizer;
+
+import java.util.EnumSet;
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import org.junit.jupiter.api.Test;
+
+import opennlp.tools.langdetect.Language;
+import opennlp.tools.langdetect.LanguageDetector;
+import opennlp.tools.stemmer.snowball.SnowballStemmer;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertSame;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+public class NormalizationProfilesTest {
+
+ @Test
+ void testEnglishUsesTheGenericAccentFold() {
+ final NormalizationProfile profile =
NormalizationProfiles.forLanguage("eng").orElseThrow();
+ assertEquals(SnowballStemmer.ALGORITHM.ENGLISH,
profile.stemmerAlgorithm());
+ assertSame(AccentFoldCharSequenceNormalizer.getInstance(),
profile.accentFold());
+ assertEquals(List.of(Dimension.NFC, Dimension.CASE_FOLD,
Dimension.ACCENT_FOLD, Dimension.STEM),
+ profile.matchingAnalyzer().dimensions());
+ }
+
+ @Test
+ void testProfileRejectsInvalidComponents() {
+ assertThrows(IllegalArgumentException.class,
+ () -> new NormalizationProfile(null,
SnowballStemmer.ALGORITHM.ENGLISH, null));
+ assertThrows(IllegalArgumentException.class,
+ () -> new NormalizationProfile("eng", null, null));
+ assertThrows(IllegalArgumentException.class,
+ () -> new NormalizationProfile(" ",
SnowballStemmer.ALGORITHM.ENGLISH, null));
+ }
+
+ @Test
+ void testTwoLetterCodeResolvesToProfile() {
+ assertEquals(SnowballStemmer.ALGORITHM.GERMAN,
+
NormalizationProfiles.forLanguage("de").orElseThrow().stemmerAlgorithm());
+ }
+
+ @Test
+ void testNorwegianWrittenStandardsResolveToTheNorwegianProfile() {
+ // "nb" (Bokmal) and "nn" (Nynorsk) are the standard modern written codes;
both convert to the
+ // ISO 639-3 codes "nob"/"nno", which must resolve to Norwegian even
though the registry also
+ // keys the "nor" macrolanguage. Without the aliases, forLanguage("nb")
returns empty -- and
+ // detect()'s "nob"/"nno" output gets no profile for a language the
registry claims to support.
+ assertEquals(SnowballStemmer.ALGORITHM.NORWEGIAN,
+
NormalizationProfiles.forLanguage("nb").orElseThrow().stemmerAlgorithm());
+ assertEquals(SnowballStemmer.ALGORITHM.NORWEGIAN,
+
NormalizationProfiles.forLanguage("nn").orElseThrow().stemmerAlgorithm());
+ assertTrue(NormalizationProfiles.forLanguage("nob").isPresent());
+ assertTrue(NormalizationProfiles.forLanguage("nno").isPresent());
+ }
+
+ @Test
+ void testGermanUsesTheGermanSpecificFold() {
+ final NormalizationProfile profile =
NormalizationProfiles.forLanguage("deu").orElseThrow();
+ assertSame(GermanUmlautCharSequenceNormalizer.getInstance(),
profile.accentFold());
+ assertEquals(List.of(Dimension.NFC, Dimension.CASE_FOLD,
Dimension.ACCENT_FOLD, Dimension.STEM),
+ profile.matchingAnalyzer().dimensions());
+ }
+
+ @Test
+ void testRomanceLanguagesUseTheGenericFold() {
+ for (final String language : List.of("fra", "spa", "por", "ita", "cat")) {
+ assertSame(AccentFoldCharSequenceNormalizer.getInstance(),
+
NormalizationProfiles.forLanguage(language).orElseThrow().accentFold());
+ }
+ }
+
+ @Test
+ void testNordicLanguageHasNoFold() {
+ final NormalizationProfile swedish =
NormalizationProfiles.forLanguage("swe").orElseThrow();
+ assertNull(swedish.accentFold());
+ assertEquals(List.of(Dimension.NFC, Dimension.CASE_FOLD, Dimension.STEM),
+ swedish.matchingAnalyzer().dimensions());
+ }
+
+ @Test
+ void testUnsupportedLanguageIsEmpty() {
+ assertTrue(NormalizationProfiles.forLanguage("jpn").isEmpty());
+ assertTrue(NormalizationProfiles.forLanguage("zzz").isEmpty());
+ }
+
+ @Test
+ void testMatchingAnalyzerStemsThroughTheChain() {
+ final NormalizationProfile english =
NormalizationProfiles.forLanguage("eng").orElseThrow();
+ assertEquals("cat",
english.matchingAnalyzer().analyze("Cats").get(0).normalized());
+ }
+
+ @Test
+ void testDetectDispatchesThroughTheDetector() {
+ final LanguageDetector detector = new LanguageDetector() {
+ @Override
+ public Language[] predictLanguages(CharSequence content) {
+ return new Language[] {new Language("deu")};
+ }
+
+ @Override
+ public Language predictLanguage(CharSequence content) {
+ return new Language("deu");
+ }
+
+ @Override
+ public String[] getSupportedLanguages() {
+ return new String[] {"deu"};
+ }
+ };
+ final NormalizationProfile profile =
+ NormalizationProfiles.detect("Guten Tag", detector).orElseThrow();
+ assertEquals(SnowballStemmer.ALGORITHM.GERMAN, profile.stemmerAlgorithm());
+ }
+
+ @Test
+ void testDetectUnsupportedLanguageIsEmpty() {
+ final LanguageDetector detector = new LanguageDetector() {
+ @Override
+ public Language[] predictLanguages(CharSequence content) {
+ return new Language[] {new Language("jpn")};
+ }
+
+ @Override
+ public Language predictLanguage(CharSequence content) {
+ return new Language("jpn");
+ }
+
+ @Override
+ public String[] getSupportedLanguages() {
+ return new String[] {"jpn"};
+ }
+ };
+ assertTrue(NormalizationProfiles.detect("text", detector).isEmpty());
+ }
+
+ @Test
+ void testSupportedLanguagesCoverEverySnowballLanguage() {
+ // Every Snowball algorithm that names a language has a profile; PORTER is
an English-only
+ // algorithm variant, not a language, so it is the sole expected omission.
Deriving the
+ // expectation from the enum makes this fail loudly if a future algorithm
is added unmapped.
+ final Set<SnowballStemmer.ALGORITHM> covered =
NormalizationProfiles.supportedLanguages().stream()
+ .map(code ->
NormalizationProfiles.forLanguage(code).orElseThrow().stemmerAlgorithm())
+ .collect(Collectors.toCollection(() ->
EnumSet.noneOf(SnowballStemmer.ALGORITHM.class)));
+
assertEquals(EnumSet.complementOf(EnumSet.of(SnowballStemmer.ALGORITHM.PORTER)),
covered);
+ // The three Norwegian written codes share the single NORWEGIAN algorithm.
+
assertTrue(NormalizationProfiles.supportedLanguages().containsAll(List.of("nob",
"nno", "nor")));
+ }
+
+ @Test
+ void testTwoLetterCodeWithNoIso3FallsBackToRawLookup() {
+ // A two-letter code with no ISO 639-3 mapping makes getISO3Language()
throw
+ // MissingResourceException; forLanguage() must catch it and fall through
to a raw lookup
+ // (which finds nothing here) rather than propagating the exception.
+ assertTrue(NormalizationProfiles.forLanguage("qq").isEmpty());
+ }
+
+ @Test
+ void testForLanguageRejectsNull() {
+ assertThrows(IllegalArgumentException.class, () ->
NormalizationProfiles.forLanguage(null));
+ }
+
+ @Test
+ void testDetectRejectsNull() {
+ final LanguageDetector detector = new LanguageDetector() {
+ @Override
+ public Language[] predictLanguages(CharSequence content) {
+ return new Language[0];
+ }
+
+ @Override
+ public Language predictLanguage(CharSequence content) {
+ return new Language("eng");
+ }
+
+ @Override
+ public String[] getSupportedLanguages() {
+ return new String[0];
+ }
+ };
+ assertThrows(IllegalArgumentException.class, () ->
NormalizationProfiles.detect(null, detector));
+ assertThrows(IllegalArgumentException.class, () ->
NormalizationProfiles.detect("text", null));
+ }
+}
