This is an automated email from the ASF dual-hosted git repository.
rzo1 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/opennlp.git
The following commit(s) were added to refs/heads/main by this push:
new 4f22baa93 GH-1464: Release the per-thread stemmer state of a
TermAnalyzer and reuse the matching analyzer (#1465)
4f22baa93 is described below
commit 4f22baa93eef99fc34ba7e001e6731e4cb57847b
Author: Richard Zowalla <[email protected]>
AuthorDate: Sun Oct 11 18:30:22 2026 +0200
GH-1464: Release the per-thread stemmer state of a TermAnalyzer and reuse
the matching analyzer (#1465)
---
.../util/normalizer/NormalizationProfile.java | 112 +++++++++++++++---
.../tools/util/normalizer/TermAnalyzer.java | 19 ++++
.../util/normalizer/NormalizationProfilesTest.java | 52 +++++++++
.../TermAnalyzerThreadLocalStateTest.java | 126 +++++++++++++++++++++
4 files changed, 292 insertions(+), 17 deletions(-)
diff --git
a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/NormalizationProfile.java
b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/NormalizationProfile.java
index 5f0d1c855..0de2c038f 100644
---
a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/NormalizationProfile.java
+++
b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/NormalizationProfile.java
@@ -16,6 +16,8 @@
*/
package opennlp.tools.util.normalizer;
+import java.util.Objects;
+
import opennlp.tools.stemmer.Stemmer;
import opennlp.tools.stemmer.StemmerFactory;
import opennlp.tools.stemmer.snowball.SnowballStemmer;
@@ -37,26 +39,59 @@ import opennlp.tools.util.ParamChecks;
* is a search-recall choice, not a statement of linguistic correctness;
callers can build a
* {@link TermAnalyzer} directly to override it.</p>
*
- * @param language The language, as an ISO 639-3 code (for example
{@code "eng"}). Must
- * not be {@code null} or blank.
- * @param stemmerAlgorithm The Snowball algorithm for the language. Must not
be {@code null}.
- * @param accentFold The diacritic fold for the language, or {@code
null} for none.
+ * <p>A profile is immutable and thread-safe. Two profiles are equal when
their language, stemmer
+ * algorithm and diacritic fold are equal.</p>
*/
-public record NormalizationProfile(String language, SnowballStemmer.ALGORITHM
stemmerAlgorithm,
- CharSequenceNormalizer accentFold) {
+public final class NormalizationProfile {
+
+ private final String language;
+ private final SnowballStemmer.ALGORITHM stemmerAlgorithm;
+ private final CharSequenceNormalizer accentFold;
+
+ /** The matching analyzer, built on the first call to {@link
#matchingAnalyzer()}. */
+ private volatile TermAnalyzer matchingAnalyzer;
/**
- * Validates the components.
+ * Creates a profile.
*
+ * @param language The language, as an ISO 639-3 code (for example
{@code "eng"}). Must
+ * not be {@code null} or blank.
+ * @param stemmerAlgorithm The Snowball algorithm for the language. Must not
be {@code null}.
+ * @param accentFold The diacritic fold for the language, or {@code
null} for none.
* @throws IllegalArgumentException if {@code language} or {@code
stemmerAlgorithm} is
* {@code null}, or if {@code language} is blank.
*/
- public NormalizationProfile {
+ public NormalizationProfile(String language, SnowballStemmer.ALGORITHM
stemmerAlgorithm,
+ CharSequenceNormalizer accentFold) {
ParamChecks.requireNonNullArg(language, "language");
ParamChecks.requireNonNullArg(stemmerAlgorithm, "stemmerAlgorithm");
if (language.isBlank()) {
throw new IllegalArgumentException("language must not be blank");
}
+ this.language = language;
+ this.stemmerAlgorithm = stemmerAlgorithm;
+ this.accentFold = accentFold;
+ }
+
+ /**
+ * {@return the language, as an ISO 639-3 code}
+ */
+ public String language() {
+ return language;
+ }
+
+ /**
+ * {@return the Snowball algorithm for the language}
+ */
+ public SnowballStemmer.ALGORITHM stemmerAlgorithm() {
+ return stemmerAlgorithm;
+ }
+
+ /**
+ * {@return the diacritic fold for the language, or {@code null} for none}
+ */
+ public CharSequenceNormalizer accentFold() {
+ return accentFold;
}
/**
@@ -77,19 +112,62 @@ public record NormalizationProfile(String language,
SnowballStemmer.ALGORITHM st
}
/**
- * Returns a matching analyzer for this language: NFC, case folding, the
language's
- * {@linkplain #accentFold() diacritic fold} when it has one, then stemming.
The returned
- * analyzer is thread-safe, and repeated words resolve from a bounded
per-thread stem cache
- * instead of being re-stemmed. The cache is keyed to the thread, so build
the analyzer once
- * and reuse it from threads that live across many calls, such as a fixed
platform-thread pool.
+ * Returns the matching analyzer for this language: NFC, case folding, the
language's
+ * {@linkplain #accentFold() diacritic fold} when it has one, then stemming.
The analyzer is
+ * built on the first call, and every call on this profile returns the same
instance. It is
+ * thread-safe, and repeated words resolve from a bounded per-thread stem
cache instead of being
+ * re-stemmed; the cache pays off on threads that live across many calls,
such as a fixed
+ * platform-thread pool. Call {@link TermAnalyzer#clearThreadLocalState()}
when a pooled thread
+ * no longer uses the analyzer.
*
* @return the analyzer.
*/
public TermAnalyzer matchingAnalyzer() {
- final TermAnalyzer.Builder builder =
TermAnalyzer.builder().nfc().caseFold();
- if (accentFold != null) {
- builder.transform(Dimension.ACCENT_FOLD, accentFold);
+ TermAnalyzer analyzer = matchingAnalyzer;
+ if (analyzer == null) {
+ synchronized (this) {
+ analyzer = matchingAnalyzer;
+ if (analyzer == null) {
+ final TermAnalyzer.Builder builder =
TermAnalyzer.builder().nfc().caseFold();
+ if (accentFold != null) {
+ builder.transform(Dimension.ACCENT_FOLD, accentFold);
+ }
+ analyzer = builder.stem(stemmerFactory()).build();
+ matchingAnalyzer = analyzer;
+ }
+ }
}
- return builder.stem(stemmerFactory()).build();
+ return analyzer;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ return obj instanceof NormalizationProfile other
+ && language.equals(other.language)
+ && stemmerAlgorithm == other.stemmerAlgorithm
+ && Objects.equals(accentFold, other.accentFold);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public int hashCode() {
+ return Objects.hash(language, stemmerAlgorithm, accentFold);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public String toString() {
+ return "NormalizationProfile[language=" + language + ", stemmerAlgorithm="
+ stemmerAlgorithm
+ + ", accentFold=" + accentFold + "]";
}
}
diff --git
a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/TermAnalyzer.java
b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/TermAnalyzer.java
index 02fb1a0ae..2c6faa895 100644
---
a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/TermAnalyzer.java
+++
b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/TermAnalyzer.java
@@ -50,6 +50,10 @@ import opennlp.tools.util.Span;
* the factory mints stateful stemmers, and repeated words resolve from a
bounded per-thread cache
* instead of being re-stemmed. A raw {@link Stemmer} passed to {@link
Builder#stem(Stemmer)} is
* only safe when that stemmer is itself thread-safe or the analyzer is
confined to one thread.</p>
+ *
+ * <p>Long-running environments such as application containers should call
+ * {@link #clearThreadLocalState()} when a pooled thread no longer uses an
analyzer whose stemmer is
+ * a {@link CachingStemmer}.</p>
*/
public final class TermAnalyzer {
@@ -145,6 +149,21 @@ public final class TermAnalyzer {
return chain;
}
+ /**
+ * Removes the calling thread's state of the analyzer's stemmer to prevent
classloader leaks in
+ * container environments. Call when the thread is returned to a pool or the
analyzer is no
+ * longer needed. The state is held when the stemmer is a {@link
CachingStemmer}, whether created
+ * by {@link Builder#stem(StemmerFactory)} or passed to {@link
Builder#stem(Stemmer)}; for any
+ * other stemmer, or without one, this method has no effect.
+ *
+ * @see CachingStemmer#clearThreadLocalState()
+ */
+ public void clearThreadLocalState() {
+ if (stemmer instanceof CachingStemmer caching) {
+ caching.clearThreadLocalState();
+ }
+ }
+
/**
* {@return the last configured dimension in pipeline order, or {@link
Dimension#ORIGINAL} when
* none are configured} This is the layer {@link Term#normalized()} reports.
diff --git
a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/NormalizationProfilesTest.java
b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/NormalizationProfilesTest.java
index f344af918..0eead3d61 100644
---
a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/NormalizationProfilesTest.java
+++
b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/NormalizationProfilesTest.java
@@ -16,6 +16,7 @@
*/
package opennlp.tools.util.normalizer;
+import java.util.ArrayList;
import java.util.EnumSet;
import java.util.List;
import java.util.Set;
@@ -26,6 +27,7 @@ import java.util.stream.Collectors;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.CsvSource;
import org.junit.jupiter.params.provider.ValueSource;
import opennlp.tools.langdetect.Language;
@@ -33,6 +35,7 @@ import opennlp.tools.langdetect.LanguageDetector;
import opennlp.tools.stemmer.snowball.SnowballStemmer;
import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotEquals;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertSame;
import static org.junit.jupiter.api.Assertions.assertThrows;
@@ -235,4 +238,53 @@ public class NormalizationProfilesTest {
}
}
}
+
+ @ParameterizedTest
+ @ValueSource(strings = {"eng", "deu", "swe", "tur"})
+ void testMatchingAnalyzerIsBuiltOncePerProfile(String language) {
+ final NormalizationProfile profile =
NormalizationProfiles.forLanguage(language).orElseThrow();
+ assertSame(profile.matchingAnalyzer(), profile.matchingAnalyzer());
+ }
+
+ @Test
+ void testMatchingAnalyzerIsSharedAcrossThreads() throws Exception {
+ final NormalizationProfile profile =
+ new NormalizationProfile("eng", SnowballStemmer.ALGORITHM.ENGLISH,
null);
+ try (ExecutorService pool = Executors.newFixedThreadPool(8)) {
+ final List<Future<TermAnalyzer>> tasks = new ArrayList<>();
+ for (int i = 0; i < 32; i++) {
+ tasks.add(pool.submit(profile::matchingAnalyzer));
+ }
+ final TermAnalyzer first = tasks.get(0).get(30,
java.util.concurrent.TimeUnit.SECONDS);
+ for (final Future<TermAnalyzer> task : tasks) {
+ assertSame(first, task.get(30, java.util.concurrent.TimeUnit.SECONDS));
+ }
+ }
+ }
+
+ @ParameterizedTest
+ @CsvSource({"eng, Cats, cat", "eng, Running, run", "deu, Häuser, haus",
"fra, Écoles, ecol"})
+ void testReusedMatchingAnalyzerStillMatches(String language, String word,
String expected) {
+ final NormalizationProfile profile =
NormalizationProfiles.forLanguage(language).orElseThrow();
+ final TermAnalyzer analyzer = profile.matchingAnalyzer();
+ final String first = analyzer.analyze(word).get(0).normalized();
+ analyzer.clearThreadLocalState();
+ assertEquals(first,
profile.matchingAnalyzer().analyze(word).get(0).normalized());
+ assertEquals(expected, first);
+ }
+
+ @Test
+ void testProfilesAreEqualByComponents() {
+ final NormalizationProfile a = new NormalizationProfile("eng",
SnowballStemmer.ALGORITHM.ENGLISH,
+ AccentFoldCharSequenceNormalizer.getInstance());
+ final NormalizationProfile b = new NormalizationProfile("eng",
SnowballStemmer.ALGORITHM.ENGLISH,
+ AccentFoldCharSequenceNormalizer.getInstance());
+ a.matchingAnalyzer();
+ assertEquals(a, b);
+ assertEquals(a.hashCode(), b.hashCode());
+ assertEquals(a, NormalizationProfiles.forLanguage("eng").orElseThrow());
+ assertNotEquals(a, new NormalizationProfile("eng",
SnowballStemmer.ALGORITHM.PORTER,
+ AccentFoldCharSequenceNormalizer.getInstance()));
+ assertNotEquals(a, new NormalizationProfile("eng",
SnowballStemmer.ALGORITHM.ENGLISH, null));
+ }
}
diff --git
a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/TermAnalyzerThreadLocalStateTest.java
b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/TermAnalyzerThreadLocalStateTest.java
new file mode 100644
index 000000000..3671ff768
--- /dev/null
+++
b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/TermAnalyzerThreadLocalStateTest.java
@@ -0,0 +1,126 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.tools.util.normalizer;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+
+import opennlp.tools.stemmer.CachingStemmer;
+import opennlp.tools.stemmer.PorterStemmer;
+import opennlp.tools.stemmer.Stemmer;
+import opennlp.tools.stemmer.StemmerFactory;
+import opennlp.tools.stemmer.snowball.SnowballStemmer;
+
+import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+class TermAnalyzerThreadLocalStateTest {
+
+ /** A factory that counts the delegates it mints and the words they stem. */
+ private static final class CountingFactory implements StemmerFactory {
+
+ private final AtomicInteger minted = new AtomicInteger();
+ private final AtomicInteger stemmed = new AtomicInteger();
+
+ @Override
+ public Stemmer newStemmer() {
+ minted.incrementAndGet();
+ return word -> {
+ stemmed.incrementAndGet();
+ return word.toString() + "-s";
+ };
+ }
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {"dog", "cat", "running"})
+ void testClearReleasesAWorkerThreadsDelegate(String word) throws Exception {
+ final CountingFactory factory = new CountingFactory();
+ final TermAnalyzer analyzer = TermAnalyzer.builder().stem(factory).build();
+ // The first stemming thread owns a retained state; a pool thread holds
per-thread state.
+ analyzer.analyze(word);
+ final ExecutorService pool = Executors.newSingleThreadExecutor();
+ try {
+ pool.submit(() -> analyzer.analyze(word)).get();
+ final int beforeClear = factory.minted.get();
+ pool.submit(() -> analyzer.analyze(word)).get();
+ assertEquals(beforeClear, factory.minted.get(), "the worker reuses its
delegate");
+
+ pool.submit(analyzer::clearThreadLocalState).get();
+ final List<Term> terms = pool.submit(() -> analyzer.analyze(word)).get();
+ assertEquals(word + "-s", terms.get(0).normalized());
+ assertEquals(beforeClear + 1, factory.minted.get(),
+ "a fresh delegate is minted after the clear");
+ } finally {
+ pool.shutdownNow();
+ }
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {"dog", "cat", "running"})
+ void testClearEmptiesTheOwnerThreadsCache(String word) {
+ final CountingFactory factory = new CountingFactory();
+ final TermAnalyzer analyzer = TermAnalyzer.builder().stem(factory,
16).build();
+ analyzer.analyze(word);
+ analyzer.analyze(word);
+ assertEquals(1, factory.stemmed.get(), "the second lookup is served from
the cache");
+
+ analyzer.clearThreadLocalState();
+ assertEquals(word + "-s", analyzer.analyze(word).get(0).normalized());
+ assertEquals(2, factory.stemmed.get(), "the clear emptied the cache");
+ }
+
+ @Test
+ void testClearReleasesAPassedCachingStemmer() throws Exception {
+ final CountingFactory factory = new CountingFactory();
+ final TermAnalyzer analyzer =
+ TermAnalyzer.builder().stem(new CachingStemmer(factory)).build();
+ analyzer.analyze("dog");
+ final ExecutorService pool = Executors.newSingleThreadExecutor();
+ try {
+ pool.submit(() -> analyzer.analyze("dog")).get();
+ final int beforeClear = factory.minted.get();
+ pool.submit(analyzer::clearThreadLocalState).get();
+ pool.submit(() -> analyzer.analyze("dog")).get();
+ assertEquals(beforeClear + 1, factory.minted.get());
+ } finally {
+ pool.shutdownNow();
+ }
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {"Running", "Cats", "jumped"})
+ void testClearIsANoOpWithoutACachingStemmer(String word) {
+ final List<TermAnalyzer> analyzers = new ArrayList<>();
+ analyzers.add(TermAnalyzer.builder().caseFold().build());
+ analyzers.add(TermAnalyzer.builder().caseFold().stem(new
PorterStemmer()).build());
+ analyzers.add(TermAnalyzer.builder().caseFold()
+ .stem(new SnowballStemmer(SnowballStemmer.ALGORITHM.ENGLISH)).build());
+ for (final TermAnalyzer analyzer : analyzers) {
+ final String before = analyzer.analyze(word).get(0).normalized();
+ assertDoesNotThrow(analyzer::clearThreadLocalState);
+ assertEquals(before, analyzer.analyze(word).get(0).normalized());
+ }
+ }
+}
