This is an automated email from the ASF dual-hosted git repository.
mawiesne pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/opennlp.git
The following commit(s) were added to refs/heads/main by this push:
new ef0a354cf OPENNLP-1883: Make stemmers thread-safe, add StemmerFactory
and per-thread stem caching (#1163)
ef0a354cf is described below
commit ef0a354cf11ae0ca00dcd7ab8d0d473b48e6fd8e
Author: Kristian Rickert <[email protected]>
AuthorDate: Mon Jul 20 11:15:41 2026 -0400
OPENNLP-1883: Make stemmers thread-safe, add StemmerFactory and per-thread
stem caching (#1163)
* SnowballStemmer now routes each call to a per-thread generated engine via
OwnerOrPerThreadState, the same pattern as the thread-safe *ME components,
so one instance can be shared across threads without touching the
generated Snowball code. StemmerFactory (api) captures stemmer
configuration as an immutable, thread-safe seam for stateful engines;
SharingStemmer adapts a factory into a shareable Stemmer for
implementations that are not thread-safe themselves (e.g. PorterStemmer).
NormalizationProfile.matchingAnalyzer() and TermAnalyzer are now safe to
share across threads when stemming.
* Add multi-threaded stemmer eval and expand concurrency unit tests
* Add SnowballStemmer JMH benchmark: thread-safe vs pre-patch baseline
* Add CachingStemmer: per-thread LRU memoization of word-to-stem mappings
Natural text is Zipf-distributed, so the same words are stemmed
constantly. CachingStemmer wraps a StemmerFactory with a bounded
per-thread LRU (default 1024 entries) following the same
OwnerOrPerThreadState pattern as the *ME components: no cross-thread
sharing, thread-safe regardless of the delegate. On the JMH zipf
workload the cache is a ~34x throughput multiplier; on a cache-hostile
uniform workload over 8x the cache capacity it still breaks even.
Results and a corrected JMH invocation are documented in BENCHMARKS.md.
* Route TermAnalyzer factory-based stemming through CachingStemmer
* Extract a DelegatingStemmer base to share the per-thread wrapper plumbing
* Add a stemmer manual chapter with a mirror-tested usage example via
docbkx/stemmer.xml documenting the Stemmer interface, StemmerFactory,
and the sharing and caching wrappers
---
.../main/java/opennlp/tools/stemmer/Stemmer.java | 25 +-
.../stemmer/{Stemmer.java => StemmerFactory.java} | 13 +-
opennlp-core/opennlp-runtime/BENCHMARKS.md | 84 ++++-
.../tools/stemmer/CachingStemmerBenchmark.java | 199 +++++++++++
.../stemmer/snowball/SnowballStemmerBenchmark.java | 153 +++++++++
.../java/opennlp/tools/stemmer/CachingStemmer.java | 152 +++++++++
.../opennlp/tools/stemmer/DelegatingStemmer.java | 70 ++++
.../java/opennlp/tools/stemmer/PorterStemmer.java | 7 +
.../tools/stemmer/PorterStemmerFactory.java | 17 +-
.../java/opennlp/tools/stemmer/SharingStemmer.java | 67 ++++
.../tools/stemmer/snowball/SnowballStemmer.java | 135 +++++---
.../stemmer/snowball/SnowballStemmerFactory.java | 109 ++++++
.../util/normalizer/NormalizationProfile.java | 23 +-
.../tools/util/normalizer/TermAnalyzer.java | 43 ++-
.../opennlp/tools/stemmer/CachingStemmerTest.java | 139 ++++++++
.../opennlp/tools/stemmer/StemmerFactoryTest.java | 376 +++++++++++++++++++++
.../stemmer/StemmerFactoryUsageExampleTest.java | 62 ++++
.../util/normalizer/NormalizationProfilesTest.java | 23 ++
.../tools/util/normalizer/TermAnalyzerTest.java | 28 +-
opennlp-docs/src/docbkx/opennlp.xml | 1 +
opennlp-docs/src/docbkx/stemmer.xml | 72 ++++
.../tools/eval/MultiThreadedStemmerEval.java | 175 ++++++++++
22 files changed, 1885 insertions(+), 88 deletions(-)
diff --git a/opennlp-api/src/main/java/opennlp/tools/stemmer/Stemmer.java
b/opennlp-api/src/main/java/opennlp/tools/stemmer/Stemmer.java
index 750890eb6..fb6bdcffb 100644
--- a/opennlp-api/src/main/java/opennlp/tools/stemmer/Stemmer.java
+++ b/opennlp-api/src/main/java/opennlp/tools/stemmer/Stemmer.java
@@ -17,10 +17,33 @@
package opennlp.tools.stemmer;
+import java.util.List;
+
/**
- * The stemmer is reducing a word to its stem.
+ * Reduces a word to its root form.
+ *
+ * <p>Thread safety is implementation specific.</p>
*/
public interface Stemmer {
+ /**
+ * Stems {@code word}.
+ *
+ * @param word The input word. Must not be {@code null}.
+ * @return The stemmed form.
+ * @throws IllegalArgumentException Thrown if {@code word} is {@code null}.
+ */
CharSequence stem(CharSequence word);
+
+ /**
+ * {@return all stem forms for {@code word}} Defaults to a single-element
list from
+ * {@link #stem(CharSequence)}. Dictionary-based engines may override this
to return
+ * multiple roots; delegating wrappers must forward it to preserve the full
list.
+ *
+ * @param word The input word. Must not be {@code null}.
+ * @throws IllegalArgumentException Thrown if {@code word} is {@code null}.
+ */
+ default List<CharSequence> stemAll(CharSequence word) {
+ return List.of(stem(word));
+ }
}
diff --git a/opennlp-api/src/main/java/opennlp/tools/stemmer/Stemmer.java
b/opennlp-api/src/main/java/opennlp/tools/stemmer/StemmerFactory.java
similarity index 68%
copy from opennlp-api/src/main/java/opennlp/tools/stemmer/Stemmer.java
copy to opennlp-api/src/main/java/opennlp/tools/stemmer/StemmerFactory.java
index 750890eb6..60c80804e 100644
--- a/opennlp-api/src/main/java/opennlp/tools/stemmer/Stemmer.java
+++ b/opennlp-api/src/main/java/opennlp/tools/stemmer/StemmerFactory.java
@@ -4,7 +4,7 @@
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
+ * the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
@@ -18,9 +18,14 @@
package opennlp.tools.stemmer;
/**
- * The stemmer is reducing a word to its stem.
+ * A factory that captures a stemmer configuration and mints configured {@link
Stemmer}
+ * instances on demand. Unlike {@code BaseToolFactory}, this type is not
loaded from a model
+ * manifest.
*/
-public interface Stemmer {
+public interface StemmerFactory {
- CharSequence stem(CharSequence word);
+ /**
+ * {@return a new {@link Stemmer}}
+ */
+ Stemmer newStemmer();
}
diff --git a/opennlp-core/opennlp-runtime/BENCHMARKS.md
b/opennlp-core/opennlp-runtime/BENCHMARKS.md
index 68c3ac185..21cb71a66 100644
--- a/opennlp-core/opennlp-runtime/BENCHMARKS.md
+++ b/opennlp-core/opennlp-runtime/BENCHMARKS.md
@@ -13,6 +13,8 @@ variance reporting.
| `TokenizerMEBenchmark` | TokenizerME | 3 approaches |
| `SentenceDetectorMEBenchmark` | SentenceDetectorME | 3 approaches |
| `POSTaggerMEBenchmark` | POSTaggerME | 3 approaches x 2 cache configs |
+| `SnowballStemmerBenchmark` | SnowballStemmer | 3 approaches (incl.
plain-field baseline) |
+| `CachingStemmerBenchmark` | CachingStemmer | cached vs uncached x 2
workloads |
### Approaches measured
@@ -30,32 +32,88 @@ mvn test-compile -Pjmh \
-pl opennlp-core/opennlp-runtime -am \
-Dforbiddenapis.skip=true -Dcheckstyle.skip=true
+# Materialize the test classpath once (JMH's forked JVMs inherit
+# java.class.path, which mvn exec:java does not populate, running
+# through exec:java fails with ClassNotFoundException: ForkedMain)
+mvn dependency:build-classpath -pl opennlp-core/opennlp-runtime \
+ -Pjmh -DincludeScope=test -Dmdep.outputFile=/tmp/cp.txt
+
+CP="opennlp-core/opennlp-runtime/target/classes:opennlp-core/opennlp-runtime/target/test-classes:$(cat
/tmp/cp.txt)"
+
# Run all ME benchmarks
-mvn exec:java -pl opennlp-core/opennlp-runtime \
- -Pjmh -Dexec.classpathScope=test \
- -Dexec.mainClass=org.openjdk.jmh.Main \
- -Dexec.args="opennlp.tools.*.ME*"
+java -cp "$CP" org.openjdk.jmh.Main 'opennlp.tools.*.ME*'
# Run POSTagger only (includes cacheSize param)
-mvn exec:java -pl opennlp-core/opennlp-runtime \
- -Pjmh -Dexec.classpathScope=test \
- -Dexec.mainClass=org.openjdk.jmh.Main \
- -Dexec.args="POSTaggerMEBenchmark"
+java -cp "$CP" org.openjdk.jmh.Main POSTaggerMEBenchmark
```
-### Regression testing (stock vs patched)
+### Baseline comparison
-Run the `newInstancePerCall` benchmark on both stock and patched
-builds. The throughput numbers should be within JMH's error margin.
+Run the `newInstancePerCall` benchmark on both `main` and this
+branch. The throughput numbers should be within JMH's error margin.
```bash
-# On stock (upstream/main):
+# On main:
# ... build and run as above, save output
-# On patched (feature/thread-safe-me):
+# On this branch:
# ... build and run as above, compare
```
+### SnowballStemmer results (Linux, JDK 25, 32 cores, 2 forks x 10 iterations)
+
+`SnowballStemmerBenchmark` compares the thread-safe `SnowballStemmer`
+(engine behind `OwnerOrPerThreadState`) against a baseline replica
+that keeps the engine in a plain field (not shareable).
+One op = stemming 16 English words.
+
+| Strategy | 1 thread | 8 threads | 32 threads |
+|----------|---------:|----------:|-----------:|
+| `sharedInstance` (one shared stemmer) | 560k ± 3k ops/s | 1.55M ± 0.17M |
3.16M ± 0.34M |
+| `instancePerThread` (stemmer per thread) | 509k ± 26k ops/s | 1.60M ± 0.17M
| 2.94M ± 0.11M |
+| `legacyInstancePerThread` (plain-field baseline, stemmer per thread) | 544k
± 19k ops/s | 1.46M ± 0.16M | 4.77M ± 0.39M |
+
+At 1 and 8 threads the three strategies are within (or nearly within)
+each other's error bars: the `OwnerOrPerThreadState` lookup is not
+measurable against the cost of stemming itself. Only at full
+saturation (32 threads, hyperthreaded) does the legacy plain-field
+baseline pull ahead (~1.5x): with every hardware thread busy, the
+per-call owner check plus `ThreadLocal` lookup is no longer hidden by
+memory-level parallelism. Real pipelines stem as one stage among many,
+so the saturated-microbenchmark gap is an upper bound, and the legacy
+strategy was not shareable across threads in the first place.
+
+### CachingStemmer results (same environment)
+
+`CachingStemmerBenchmark` compares a `CachingStemmer` (per-thread LRU,
+default 1024 entries, wrapping the English Snowball stemmer) against
+the uncached shared stemmer. One op = 16 tokens from a 64k-token
+stream. The `zipf` workload samples a 512-word vocabulary with 1/rank
+weights (real-text repetition; the cache holds the whole vocabulary);
+`diverse` samples an 8192-word vocabulary uniformly (8x cache
+capacity: mostly misses plus constant eviction).
+
+| Workload | Strategy | 8 threads | 32 threads |
+|----------|----------|----------:|-----------:|
+| `zipf` | `cachedShared` | 48.5M ± 0.7M ops/s | 95.4M ± 0.9M |
+| `zipf` | `uncachedShared` | 1.43M ± 0.13M | 2.75M ± 0.34M |
+| `diverse` | `cachedShared` | 1.81M ± 0.51M | 3.45M ± 0.18M |
+| `diverse` | `uncachedShared` | 1.08M ± 0.09M | 3.13M ± 0.12M |
+
+On the Zipf workload the cache is a ~34x throughput multiplier (raw
+stemming becomes a hash lookup for the dominant vocabulary). On the
+cache-hostile workload it still does not lose: the ~12% residual hit
+rate pays for the eviction overhead. The cache more than recovers the
+`OwnerOrPerThreadState` lookup cost observed in
+`SnowballStemmerBenchmark` at full saturation.
+
+The cache is keyed to the physical thread, and these runs use a fixed
+platform-thread pool whose threads live for the whole measurement. On
+a virtual-thread-per-task executor every task starts with an empty
+cache, so the multiplier only applies to repeats within one task;
+workloads that stem a handful of words per task should expect
+uncached-level throughput there.
+
### POSTagger cache impact
The `POSTaggerMEBenchmark` uses `@Param({"0", "3"})` for cache
diff --git
a/opennlp-core/opennlp-runtime/src/jmh/java/opennlp/tools/stemmer/CachingStemmerBenchmark.java
b/opennlp-core/opennlp-runtime/src/jmh/java/opennlp/tools/stemmer/CachingStemmerBenchmark.java
new file mode 100644
index 000000000..ae0050b2b
--- /dev/null
+++
b/opennlp-core/opennlp-runtime/src/jmh/java/opennlp/tools/stemmer/CachingStemmerBenchmark.java
@@ -0,0 +1,199 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.stemmer;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+import java.util.concurrent.TimeUnit;
+
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Level;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Threads;
+import org.openjdk.jmh.annotations.Warmup;
+import org.openjdk.jmh.infra.Blackhole;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
+
+import opennlp.tools.stemmer.snowball.SnowballStemmer;
+import opennlp.tools.stemmer.snowball.SnowballStemmerFactory;
+
+/**
+ * JMH benchmark for {@link CachingStemmer} against an uncached shared {@link
SnowballStemmer}.
+ *
+ * <p>Two workloads drive both strategies:</p>
+ * <ul>
+ * <li>{@code zipf}: a 64k-token stream sampled with 1/rank weights from a
512-word
+ * vocabulary. This models real text, where a small vocabulary
dominates; the default
+ * 1024-entry cache holds the whole vocabulary.</li>
+ * <li>{@code diverse}: a 64k-token stream sampled uniformly from an
8192-word vocabulary,
+ * 8x the cache capacity. This is the cache-hostile case: mostly misses
plus constant
+ * eviction, so it bounds the overhead the cache can add.</li>
+ * </ul>
+ *
+ * <p>One op stems 16 consecutive tokens from the stream; each benchmark
thread walks the stream
+ * from its own cursor.</p>
+ */
+@BenchmarkMode(Mode.Throughput)
+@OutputTimeUnit(TimeUnit.SECONDS)
+@Warmup(iterations = 5, time = 2)
+@Measurement(iterations = 10, time = 2)
+@Fork(2)
+public class CachingStemmerBenchmark {
+
+ private static final int STREAM_LENGTH = 65536;
+ private static final int WORDS_PER_OP = 16;
+
+ private static final String[] ROOTS = {
+ "run", "walk", "talk", "develop", "nation", "connect", "form", "create",
+ "act", "direct", "govern", "manage", "operate", "organize", "present",
"relate",
+ "report", "state", "structure", "test", "train", "transform",
"translate", "value",
+ "view", "wonder", "yield", "zone", "note", "mark", "place", "point"
+ };
+ private static final String[] PREFIXES = {
+ "", "re", "un", "over", "under", "out", "pre", "post",
+ "non", "anti", "de", "dis", "mis", "sub", "super", "inter"
+ };
+ private static final String[] SUFFIXES = {
+ "", "s", "ed", "ing", "er", "ers", "ation", "ations",
+ "ly", "ness", "ment", "ments", "ize", "ized", "izing", "al"
+ };
+
+ @State(Scope.Benchmark)
+ public static class WorkloadState {
+
+ @Param({"zipf", "diverse"})
+ String workload;
+
+ String[] stream;
+
+ @Setup(Level.Trial)
+ public void build() {
+ Random random = new Random(42);
+ List<String> vocabulary = new ArrayList<>();
+ if ("zipf".equals(workload)) {
+ // 32 roots x 16 suffixes = 512 unique words, sampled with 1/rank
weights.
+ for (String root : ROOTS) {
+ for (String suffix : SUFFIXES) {
+ vocabulary.add(root + suffix);
+ }
+ }
+ double[] cumulative = new double[vocabulary.size()];
+ double sum = 0;
+ for (int rank = 0; rank < vocabulary.size(); rank++) {
+ sum += 1.0 / (rank + 1);
+ cumulative[rank] = sum;
+ }
+ stream = new String[STREAM_LENGTH];
+ for (int i = 0; i < STREAM_LENGTH; i++) {
+ double r = random.nextDouble() * sum;
+ int idx = 0;
+ while (cumulative[idx] < r) {
+ idx++;
+ }
+ stream[i] = vocabulary.get(idx);
+ }
+ } else {
+ // 16 prefixes x 32 roots x 16 suffixes = 8192 unique words, sampled
uniformly.
+ for (String prefix : PREFIXES) {
+ for (String root : ROOTS) {
+ for (String suffix : SUFFIXES) {
+ vocabulary.add(prefix + root + suffix);
+ }
+ }
+ }
+ stream = new String[STREAM_LENGTH];
+ for (int i = 0; i < STREAM_LENGTH; i++) {
+ stream[i] = vocabulary.get(random.nextInt(vocabulary.size()));
+ }
+ }
+ }
+ }
+
+ @State(Scope.Benchmark)
+ public static class UncachedState {
+ Stemmer stemmer;
+
+ @Setup(Level.Trial)
+ public void create() {
+ stemmer = new SnowballStemmer(SnowballStemmer.ALGORITHM.ENGLISH);
+ }
+ }
+
+ @State(Scope.Benchmark)
+ public static class CachedState {
+ Stemmer stemmer;
+
+ @Setup(Level.Trial)
+ public void create() {
+ stemmer = new CachingStemmer(
+ new SnowballStemmerFactory(SnowballStemmer.ALGORITHM.ENGLISH));
+ }
+ }
+
+ @State(Scope.Thread)
+ public static class Cursor {
+ int position;
+
+ @Setup(Level.Trial)
+ public void randomize() {
+ position = new Random().nextInt(STREAM_LENGTH);
+ }
+ }
+
+ @Benchmark
+ @Threads(Threads.MAX)
+ public void uncachedShared(WorkloadState w, UncachedState st, Cursor cursor,
Blackhole bh) {
+ for (int i = 0; i < WORDS_PER_OP; i++) {
+ bh.consume(st.stemmer.stem(w.stream[cursor.position++ & (STREAM_LENGTH -
1)]));
+ }
+ }
+
+ @Benchmark
+ @Threads(Threads.MAX)
+ public void cachedShared(WorkloadState w, CachedState st, Cursor cursor,
Blackhole bh) {
+ for (int i = 0; i < WORDS_PER_OP; i++) {
+ bh.consume(st.stemmer.stem(w.stream[cursor.position++ & (STREAM_LENGTH -
1)]));
+ }
+ }
+
+ /**
+ * Quick local iteration only: {@code forks(0)} disables JVM fork isolation
+ * (unlike {@code mvn} with the {@code jmh} profile).
+ * Use the Maven-invoked configuration for publishable numbers.
+ */
+ public static void main(String[] args) throws Exception {
+ Options opt = new OptionsBuilder()
+ .include(CachingStemmerBenchmark.class.getSimpleName())
+ .forks(0)
+ .warmupIterations(3)
+ .measurementIterations(5)
+ .build();
+ new Runner(opt).run();
+ }
+}
diff --git
a/opennlp-core/opennlp-runtime/src/jmh/java/opennlp/tools/stemmer/snowball/SnowballStemmerBenchmark.java
b/opennlp-core/opennlp-runtime/src/jmh/java/opennlp/tools/stemmer/snowball/SnowballStemmerBenchmark.java
new file mode 100644
index 000000000..62eb09031
--- /dev/null
+++
b/opennlp-core/opennlp-runtime/src/jmh/java/opennlp/tools/stemmer/snowball/SnowballStemmerBenchmark.java
@@ -0,0 +1,153 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.stemmer.snowball;
+
+import java.util.concurrent.TimeUnit;
+
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Level;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Threads;
+import org.openjdk.jmh.annotations.Warmup;
+import org.openjdk.jmh.infra.Blackhole;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
+
+import opennlp.tools.stemmer.Stemmer;
+
+/**
+ * JMH benchmark for the thread-safe {@link SnowballStemmer} versus a baseline
replica of the
+ * previous implementation that held its generated engine in a plain field.
+ *
+ * <p>Three strategies are measured, all at {@link Threads#MAX}:</p>
+ * <ul>
+ * <li>{@code sharedInstance}: one thread-safe {@link SnowballStemmer}
shared by every thread
+ * (one thread runs on the owner fast path, the rest on {@link
ThreadLocal} state)</li>
+ * <li>{@code instancePerThread}: one thread-safe {@link SnowballStemmer}
per thread (every
+ * thread is the owner of its own instance, so this isolates the
owner-fast-path cost)</li>
+ * <li>{@code legacyInstancePerThread}: the old non-thread-safe
implementation, one per thread
+ * (the baseline; sharing it across threads would be a correctness
bug)</li>
+ * </ul>
+ */
+@BenchmarkMode(Mode.Throughput)
+@OutputTimeUnit(TimeUnit.SECONDS)
+@Warmup(iterations = 5, time = 2)
+@Measurement(iterations = 10, time = 2)
+@Fork(2)
+public class SnowballStemmerBenchmark {
+
+ private static final String[] INPUT = {
+ "running", "accompanying", "malediction", "softeners", "declining",
+ "conspiracies", "monotonically", "annotations", "internationalization",
+ "denormalization", "photographers", "responsibilities",
"acknowledgement",
+ "this", "cat", "querying"
+ };
+
+ /**
+ * Baseline replica of the previous {@code SnowballStemmer} implementation:
the generated engine
+ * lives in a plain field, so an instance must not be shared across threads.
+ */
+ static final class LegacySnowballStemmer implements Stemmer {
+
+ private final AbstractSnowballStemmer stemmer = new englishStemmer();
+
+ @Override
+ public CharSequence stem(CharSequence word) {
+ stemmer.setCurrent(word.toString());
+ stemmer.stem();
+ return stemmer.getCurrent();
+ }
+ }
+
+ @State(Scope.Benchmark)
+ public static class SharedState {
+ Stemmer stemmer;
+
+ @Setup(Level.Trial)
+ public void create() {
+ stemmer = new SnowballStemmer(SnowballStemmer.ALGORITHM.ENGLISH);
+ }
+ }
+
+ @State(Scope.Thread)
+ public static class PerThreadState {
+ Stemmer stemmer;
+
+ @Setup(Level.Trial)
+ public void create() {
+ stemmer = new SnowballStemmer(SnowballStemmer.ALGORITHM.ENGLISH);
+ }
+ }
+
+ @State(Scope.Thread)
+ public static class LegacyPerThreadState {
+ Stemmer stemmer;
+
+ @Setup(Level.Trial)
+ public void create() {
+ stemmer = new LegacySnowballStemmer();
+ }
+ }
+
+ @Benchmark
+ @Threads(Threads.MAX)
+ public void sharedInstance(SharedState st, Blackhole bh) {
+ for (String s : INPUT) {
+ bh.consume(st.stemmer.stem(s));
+ }
+ }
+
+ @Benchmark
+ @Threads(Threads.MAX)
+ public void instancePerThread(PerThreadState pt, Blackhole bh) {
+ for (String s : INPUT) {
+ bh.consume(pt.stemmer.stem(s));
+ }
+ }
+
+ @Benchmark
+ @Threads(Threads.MAX)
+ public void legacyInstancePerThread(LegacyPerThreadState pt, Blackhole bh) {
+ for (String s : INPUT) {
+ bh.consume(pt.stemmer.stem(s));
+ }
+ }
+
+ /**
+ * Quick local iteration only: {@code forks(0)} disables JVM fork isolation
+ * (unlike {@code mvn} with the {@code jmh} profile).
+ * Use the Maven-invoked configuration for publishable numbers.
+ */
+ public static void main(String[] args) throws Exception {
+ Options opt = new OptionsBuilder()
+ .include(SnowballStemmerBenchmark.class.getSimpleName())
+ .forks(0)
+ .warmupIterations(3)
+ .measurementIterations(5)
+ .build();
+ new Runner(opt).run();
+ }
+}
diff --git
a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/CachingStemmer.java
b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/CachingStemmer.java
new file mode 100644
index 000000000..b4ec524a0
--- /dev/null
+++
b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/CachingStemmer.java
@@ -0,0 +1,152 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.stemmer;
+
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.function.Supplier;
+
+import opennlp.tools.commons.ThreadSafe;
+
+/**
+ * A {@link Stemmer} that memoizes word-to-stem mappings in a bounded
per-thread LRU cache.
+ *
+ * <p>Each thread gets its own delegate stemmer (minted from the supplied
{@link StemmerFactory})
+ * and its own cache, so instances are safe to share regardless of whether the
factory's stemmers
+ * are, and memory is bounded to {@code capacity} entries per thread that
stems. Caching pays off
+ * on threads reused across many words, such as a fixed platform-thread pool;
on a
+ * virtual-thread-per-task executor every task starts with an empty cache, so
repeats are served
+ * only within one task. {@link #stemAll(CharSequence)} is forwarded to the
delegate uncached.</p>
+ *
+ * <p>Long-running environments such as application containers should call
+ * {@link #clearThreadLocalState()} when a pooled thread no longer uses this
stemmer.</p>
+ */
+@ThreadSafe
+public final class CachingStemmer extends
DelegatingStemmer<CachingStemmer.ThreadState> {
+
+ /**
+ * Covers the high-frequency vocabulary of most corpora while keeping the
per-thread footprint
+ * small.
+ */
+ public static final int DEFAULT_CAPACITY = 1024;
+
+ /**
+ * Creates a caching stemmer with the {@linkplain #DEFAULT_CAPACITY default
capacity}.
+ *
+ * @param factory The factory that mints one delegate per thread. Must not
be {@code null}.
+ * @throws IllegalArgumentException if {@code factory} is {@code null}.
+ */
+ public CachingStemmer(StemmerFactory factory) {
+ this(factory, DEFAULT_CAPACITY);
+ }
+
+ /**
+ * Creates a caching stemmer.
+ *
+ * @param factory The factory that mints one delegate per thread. Must not
be {@code null}.
+ * @param capacity The maximum number of word-to-stem entries kept per
thread; must be positive.
+ * @throws IllegalArgumentException if {@code factory} is {@code null} or
{@code capacity} is
+ * not positive.
+ */
+ public CachingStemmer(StemmerFactory factory, int capacity) {
+ super(threadStateSupplier(factory, capacity), threadState ->
threadState.cache.clear());
+ }
+
+ /**
+ * Validates the constructor arguments eagerly and returns a supplier that
mints one delegate and
+ * its per-thread cache, so invalid arguments fail at construction rather
than on first use.
+ *
+ * @param factory The factory that mints one delegate per thread. Must not
be {@code null}.
+ * @param capacity The maximum number of word-to-stem entries kept per
thread; must be positive.
+ * @return a supplier of fresh per-thread state.
+ * @throws IllegalArgumentException if {@code factory} is {@code null} or
{@code capacity} is
+ * not positive.
+ */
+ private static Supplier<ThreadState> threadStateSupplier(StemmerFactory
factory, int capacity) {
+ requireFactory(factory);
+ if (capacity <= 0) {
+ throw new IllegalArgumentException("capacity must be positive, got " +
capacity);
+ }
+ return () -> new ThreadState(factory.newStemmer(), capacity);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public CharSequence stem(CharSequence word) {
+ if (word == null) {
+ throw new IllegalArgumentException("word must not be null");
+ }
+ final ThreadState ts = state.get();
+ final String key = word.toString();
+ final String cached = ts.cache.get(key);
+ if (cached != null) {
+ return cached;
+ }
+ final String stemmed = ts.delegate.stem(key).toString();
+ ts.cache.put(key, stemmed);
+ return stemmed;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public List<CharSequence> stemAll(CharSequence word) {
+ if (word == null) {
+ throw new IllegalArgumentException("word must not be null");
+ }
+ return state.get().delegate.stemAll(word);
+ }
+
+ /**
+ * Empties the calling thread's cache while keeping its delegate, forcing
every subsequent
+ * word through a fresh stemming pass. Only the calling thread's cache is
affected. A thread
+ * that has not stemmed yet has its state initialized by this call, so
invoke it on the
+ * thread whose cache should be emptied, not from an unrelated maintenance
thread.
+ */
+ public void clearCache() {
+ state.get().cache.clear();
+ }
+
+ static final class ThreadState {
+
+ private final Stemmer delegate;
+ private final LinkedHashMap<String, String> cache;
+
+ /**
+ * Creates the per-thread delegate and its access-ordered LRU cache.
+ *
+ * @param delegate The stemmer minted for this thread.
+ * @param capacity The maximum number of entries retained before eviction.
+ */
+ private ThreadState(Stemmer delegate, int capacity) {
+ this.delegate = delegate;
+ // Access-ordered for LRU eviction.
+ this.cache = new LinkedHashMap<>((int) Math.min(capacity / 0.75d + 1.0d,
4096.0d),
+ 0.75f, true) {
+ @Override
+ protected boolean removeEldestEntry(Map.Entry<String, String> eldest) {
+ return size() > capacity;
+ }
+ };
+ }
+ }
+}
diff --git
a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/DelegatingStemmer.java
b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/DelegatingStemmer.java
new file mode 100644
index 000000000..bbec1a818
--- /dev/null
+++
b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/DelegatingStemmer.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.stemmer;
+
+import java.util.function.Consumer;
+import java.util.function.Supplier;
+
+import opennlp.tools.util.OwnerOrPerThreadState;
+
+/**
+ * Shared plumbing for {@link Stemmer} wrappers that route each call to a
per-thread payload minted
+ * from a {@link StemmerFactory}, following the {@link OwnerOrPerThreadState}
pattern of the
+ * thread-safe {@code *ME} components. Subclasses differ only in the payload
they carry per thread
+ * and in what {@link #stem(CharSequence)} does with it.
+ *
+ * @param <P> the per-thread payload: a bare delegate stemmer, or a delegate
paired with per-thread
+ * working state such as a cache.
+ */
+abstract class DelegatingStemmer<P> implements Stemmer {
+
+ /** The per-thread payload holder. */
+ protected final OwnerOrPerThreadState<P> state;
+
+ /**
+ * @param init Mints the per-thread payload on first use by the owner and
by each extra thread.
+ * @param cleanup Releases a per-thread payload when its thread clears state.
+ */
+ DelegatingStemmer(Supplier<P> init, Consumer<P> cleanup) {
+ this.state = new OwnerOrPerThreadState<>(init, cleanup);
+ }
+
+ /**
+ * Validates a factory argument before it is captured by a subclass
supplier, so a {@code null}
+ * factory fails at construction rather than on first use.
+ *
+ * @param factory The factory to validate. Must not be {@code null}.
+ * @return {@code factory}.
+ * @throws IllegalArgumentException if {@code factory} is {@code null}.
+ */
+ static StemmerFactory requireFactory(StemmerFactory factory) {
+ if (factory == null) {
+ throw new IllegalArgumentException("factory must not be null");
+ }
+ return factory;
+ }
+
+ /**
+ * Removes this thread's payload to prevent classloader leaks in container
environments. Call
+ * when the thread is returned to a pool or the stemmer is no longer needed,
mirroring
+ * {@code clearThreadLocalState()} on the thread-safe {@code *ME} components.
+ */
+ public void clearThreadLocalState() {
+ state.clearForCurrentThread();
+ }
+}
diff --git
a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/PorterStemmer.java
b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/PorterStemmer.java
index 623a2d3d2..aac73519f 100644
---
a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/PorterStemmer.java
+++
b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/PorterStemmer.java
@@ -597,8 +597,15 @@ public class PorterStemmer implements Stemmer {
/**
* Stem a word provided as a CharSequence.
* Returns the result as a CharSequence.
+ *
+ * @param word The word to stem. Must not be {@code null}.
+ * @return the stemmed word.
+ * @throws IllegalArgumentException if {@code word} is {@code null}.
*/
public CharSequence stem(CharSequence word) {
+ if (word == null) {
+ throw new IllegalArgumentException("word must not be null");
+ }
return stem(word.toString());
}
diff --git a/opennlp-api/src/main/java/opennlp/tools/stemmer/Stemmer.java
b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/PorterStemmerFactory.java
similarity index 69%
copy from opennlp-api/src/main/java/opennlp/tools/stemmer/Stemmer.java
copy to
opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/PorterStemmerFactory.java
index 750890eb6..68285109a 100644
--- a/opennlp-api/src/main/java/opennlp/tools/stemmer/Stemmer.java
+++
b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/PorterStemmerFactory.java
@@ -4,7 +4,7 @@
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
+ * the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
@@ -17,10 +17,19 @@
package opennlp.tools.stemmer;
+import opennlp.tools.commons.ThreadSafe;
+
/**
- * The stemmer is reducing a word to its stem.
+ * A thread-safe factory for {@link PorterStemmer} instances.
*/
-public interface Stemmer {
+@ThreadSafe
+public class PorterStemmerFactory implements StemmerFactory {
- CharSequence stem(CharSequence word);
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public Stemmer newStemmer() {
+ return new PorterStemmer();
+ }
}
diff --git
a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/SharingStemmer.java
b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/SharingStemmer.java
new file mode 100644
index 000000000..59e8d1508
--- /dev/null
+++
b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/SharingStemmer.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.stemmer;
+
+import java.util.List;
+
+import opennlp.tools.commons.ThreadSafe;
+import opennlp.tools.util.OwnerOrPerThreadState;
+
+/**
+ * A {@link Stemmer} that is safe to share across threads by routing each call
to a per-thread
+ * delegate minted from a {@link StemmerFactory}. Use it to share a stemmer
whose implementation is
+ * <em>not</em> itself thread-safe, such as {@link PorterStemmer}; thread-safe
implementations like
+ * {@code SnowballStemmer} do not need this wrapper.
+ *
+ * <p>The first thread uses a single owner delegate without touching {@link
ThreadLocal} until a
+ * second thread appears, matching the {@link OwnerOrPerThreadState} pattern
used by the
+ * thread-safe {@code *ME} components.</p>
+ */
+@ThreadSafe
+public final class SharingStemmer extends DelegatingStemmer<Stemmer> {
+
+ /**
+ * @param factory The factory that mints per-thread delegates. Must not be
{@code null}.
+ * @throws IllegalArgumentException if {@code factory} is {@code null}.
+ */
+ public SharingStemmer(StemmerFactory factory) {
+ super(requireFactory(factory)::newStemmer, stemmer -> { });
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public CharSequence stem(CharSequence word) {
+ if (word == null) {
+ throw new IllegalArgumentException("word must not be null");
+ }
+ return state.get().stem(word);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public List<CharSequence> stemAll(CharSequence word) {
+ if (word == null) {
+ throw new IllegalArgumentException("word must not be null");
+ }
+ return state.get().stemAll(word);
+ }
+}
diff --git
a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/snowball/SnowballStemmer.java
b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/snowball/SnowballStemmer.java
index 7b8b1636a..bc634f1c4 100644
---
a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/snowball/SnowballStemmer.java
+++
b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/snowball/SnowballStemmer.java
@@ -17,76 +17,99 @@
package opennlp.tools.stemmer.snowball;
+import java.util.function.Supplier;
+
+import opennlp.tools.commons.ThreadSafe;
+import opennlp.tools.stemmer.SharingStemmer;
import opennlp.tools.stemmer.Stemmer;
+/**
+ * A {@link Stemmer} backed by the generated Snowball engines.
+ *
+ * <p>The generated engines hold mutable per-call buffers, so this class
routes each call to a
+ * per-thread engine, the same pattern the thread-safe {@code *ME} components
use. A single
+ * {@code SnowballStemmer} instance is therefore safe to share across
threads.</p>
+ *
+ * <p>As with the {@code *ME} components, long-running environments such as
application
+ * containers should call {@link #clearThreadLocalState()} when a pooled
thread no longer uses
+ * this stemmer; otherwise the thread retains its engine until the instance is
unreachable,
+ * which can pin the defining classloader on redeploys.</p>
+ */
+@ThreadSafe
public class SnowballStemmer implements Stemmer {
- private final AbstractSnowballStemmer stemmer;
- private final int repeat;
+ private final SharingStemmer sharing;
+ /**
+ * Creates a stemmer for the given algorithm and repeat count.
+ *
+ * @param algorithm The Snowball algorithm. Must not be {@code null}.
+ * @param repeat How many times to apply the stemmer per word; must be
positive.
+ * @throws IllegalArgumentException Thrown if {@code algorithm} is {@code
null} or
+ * {@code repeat} is not positive.
+ */
public SnowballStemmer(ALGORITHM algorithm, int repeat) {
- this.repeat = repeat;
-
- if (ALGORITHM.ARABIC.equals(algorithm)) {
- stemmer = new arabicStemmer();
- } else if (ALGORITHM.DANISH.equals(algorithm)) {
- stemmer = new danishStemmer();
- } else if (ALGORITHM.DUTCH.equals(algorithm)) {
- stemmer = new dutchStemmer();
- } else if (ALGORITHM.CATALAN.equals(algorithm)) {
- stemmer = new catalanStemmer();
- } else if (ALGORITHM.ENGLISH.equals(algorithm)) {
- stemmer = new englishStemmer();
- } else if (ALGORITHM.FINNISH.equals(algorithm)) {
- stemmer = new finnishStemmer();
- } else if (ALGORITHM.FRENCH.equals(algorithm)) {
- stemmer = new frenchStemmer();
- } else if (ALGORITHM.GERMAN.equals(algorithm)) {
- stemmer = new germanStemmer();
- } else if (ALGORITHM.GREEK.equals(algorithm)) {
- stemmer = new greekStemmer();
- } else if (ALGORITHM.HUNGARIAN.equals(algorithm)) {
- stemmer = new hungarianStemmer();
- } else if (ALGORITHM.INDONESIAN.equals(algorithm)) {
- stemmer = new indonesianStemmer();
- } else if (ALGORITHM.IRISH.equals(algorithm)) {
- stemmer = new irishStemmer();
- } else if (ALGORITHM.ITALIAN.equals(algorithm)) {
- stemmer = new italianStemmer();
- } else if (ALGORITHM.NORWEGIAN.equals(algorithm)) {
- stemmer = new norwegianStemmer();
- } else if (ALGORITHM.PORTER.equals(algorithm)) {
- stemmer = new porterStemmer();
- } else if (ALGORITHM.PORTUGUESE.equals(algorithm)) {
- stemmer = new portugueseStemmer();
- } else if (ALGORITHM.ROMANIAN.equals(algorithm)) {
- stemmer = new romanianStemmer();
- } else if (ALGORITHM.RUSSIAN.equals(algorithm)) {
- stemmer = new russianStemmer();
- } else if (ALGORITHM.SPANISH.equals(algorithm)) {
- stemmer = new spanishStemmer();
- } else if (ALGORITHM.SWEDISH.equals(algorithm)) {
- stemmer = new swedishStemmer();
- } else if (ALGORITHM.TURKISH.equals(algorithm)) {
- stemmer = new turkishStemmer();
- } else {
- throw new IllegalStateException("Unexpected stemmer algorithm: " +
algorithm);
- }
+ this.sharing = new SharingStemmer(new SnowballStemmerFactory(algorithm,
repeat));
}
+ /**
+ * Creates a stemmer for the given algorithm with {@code repeat = 1}.
+ *
+ * @param algorithm The Snowball algorithm. Must not be {@code null}.
+ * @throws IllegalArgumentException Thrown if {@code algorithm} is {@code
null}.
+ */
public SnowballStemmer(ALGORITHM algorithm) {
this(algorithm, 1);
}
- public CharSequence stem(CharSequence word) {
-
- stemmer.setCurrent(word.toString());
+ /**
+ * Returns a supplier of a fresh Snowball engine for the given algorithm,
shared with
+ * {@link SnowballStemmerFactory} whose products embed one engine directly.
+ *
+ * @param algorithm The Snowball algorithm.
+ * @return a supplier that mints a new engine on each call.
+ */
+ static Supplier<AbstractSnowballStemmer> engineFor(ALGORITHM algorithm) {
+ return switch (algorithm) {
+ case ARABIC -> arabicStemmer::new;
+ case DANISH -> danishStemmer::new;
+ case DUTCH -> dutchStemmer::new;
+ case CATALAN -> catalanStemmer::new;
+ case ENGLISH -> englishStemmer::new;
+ case FINNISH -> finnishStemmer::new;
+ case FRENCH -> frenchStemmer::new;
+ case GERMAN -> germanStemmer::new;
+ case GREEK -> greekStemmer::new;
+ case HUNGARIAN -> hungarianStemmer::new;
+ case INDONESIAN -> indonesianStemmer::new;
+ case IRISH -> irishStemmer::new;
+ case ITALIAN -> italianStemmer::new;
+ case NORWEGIAN -> norwegianStemmer::new;
+ case PORTER -> porterStemmer::new;
+ case PORTUGUESE -> portugueseStemmer::new;
+ case ROMANIAN -> romanianStemmer::new;
+ case RUSSIAN -> russianStemmer::new;
+ case SPANISH -> spanishStemmer::new;
+ case SWEDISH -> swedishStemmer::new;
+ case TURKISH -> turkishStemmer::new;
+ };
+ }
- for (int i = 0; i < repeat; i++) {
- stemmer.stem();
- }
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public CharSequence stem(CharSequence word) {
+ return sharing.stem(word);
+ }
- return stemmer.getCurrent();
+ /**
+ * Removes this thread's engine to prevent classloader leaks in container
environments. Call
+ * when the thread is returned to a pool or the stemmer is no longer needed,
mirroring
+ * {@code clearThreadLocalState()} on the thread-safe {@code *ME} components.
+ */
+ public void clearThreadLocalState() {
+ sharing.clearThreadLocalState();
}
public enum ALGORITHM {
diff --git
a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/snowball/SnowballStemmerFactory.java
b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/snowball/SnowballStemmerFactory.java
new file mode 100644
index 000000000..69a674825
--- /dev/null
+++
b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/snowball/SnowballStemmerFactory.java
@@ -0,0 +1,109 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.stemmer.snowball;
+
+import opennlp.tools.commons.ThreadSafe;
+import opennlp.tools.stemmer.Stemmer;
+import opennlp.tools.stemmer.StemmerFactory;
+
+/**
+ * A thread-safe factory that captures a Snowball stemmer configuration for
APIs that accept a
+ * {@link StemmerFactory}.
+ *
+ * <p>Use this factory for the classic one-stemmer-per-thread pattern: {@link
#newStemmer()}
+ * returns a plain, thread-confined stemmer without the per-call thread
routing of the
+ * shareable {@link SnowballStemmer}.</p>
+ */
+@ThreadSafe
+public class SnowballStemmerFactory implements StemmerFactory {
+
+ private final SnowballStemmer.ALGORITHM algorithm;
+ private final int repeat;
+
+ /**
+ * Creates a factory with {@code repeat = 1}.
+ *
+ * @param algorithm The Snowball algorithm. Must not be {@code null}.
+ * @throws IllegalArgumentException if {@code algorithm} is {@code null}.
+ */
+ public SnowballStemmerFactory(SnowballStemmer.ALGORITHM algorithm) {
+ this(algorithm, 1);
+ }
+
+ /**
+ * Creates a factory over the given algorithm and repeat count.
+ *
+ * @param algorithm The Snowball algorithm. Must not be {@code null}.
+ * @param repeat How many times to apply the stemmer per word; must be
positive.
+ * @throws IllegalArgumentException if {@code algorithm} is {@code null} or
{@code repeat}
+ * is not positive.
+ */
+ public SnowballStemmerFactory(SnowballStemmer.ALGORITHM algorithm, int
repeat) {
+ if (algorithm == null) {
+ throw new IllegalArgumentException("algorithm must not be null");
+ }
+ if (repeat <= 0) {
+ throw new IllegalArgumentException("repeat must be positive, got " +
repeat);
+ }
+ this.algorithm = algorithm;
+ this.repeat = repeat;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public Stemmer newStemmer() {
+ return new ConfinedStemmer(SnowballStemmer.engineFor(algorithm).get(),
repeat);
+ }
+
+ /** A plain, thread-confined stemmer that applies one Snowball engine
directly; not thread-safe. */
+ private static final class ConfinedStemmer implements Stemmer {
+
+ private final AbstractSnowballStemmer engine;
+ private final int repeat;
+
+ /**
+ * Creates a thread-confined stemmer over the given engine.
+ *
+ * @param engine The Snowball engine to apply.
+ * @param repeat How many times to apply the engine per word.
+ */
+ private ConfinedStemmer(AbstractSnowballStemmer engine, int repeat) {
+ this.engine = engine;
+ this.repeat = repeat;
+ }
+
+ /**
+ * {@inheritDoc}
+ *
+ * @throws IllegalArgumentException if {@code word} is {@code null}.
+ */
+ @Override
+ public CharSequence stem(CharSequence word) {
+ if (word == null) {
+ throw new IllegalArgumentException("word must not be null");
+ }
+ engine.setCurrent(word.toString());
+ for (int i = 0; i < repeat; i++) {
+ engine.stem();
+ }
+ return engine.getCurrent();
+ }
+ }
+}
diff --git
a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/NormalizationProfile.java
b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/NormalizationProfile.java
index e7a47477e..57283d34d 100644
---
a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/NormalizationProfile.java
+++
b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/NormalizationProfile.java
@@ -17,7 +17,9 @@
package opennlp.tools.util.normalizer;
import opennlp.tools.stemmer.Stemmer;
+import opennlp.tools.stemmer.StemmerFactory;
import opennlp.tools.stemmer.snowball.SnowballStemmer;
+import opennlp.tools.stemmer.snowball.SnowballStemmerFactory;
/**
* Per-language normalization settings, mirroring how OpenNLP already selects
a Snowball stemmer by
@@ -61,8 +63,17 @@ public record NormalizationProfile(String language,
SnowballStemmer.ALGORITHM st
}
/**
- * {@return a new {@link Stemmer} for this language} A fresh instance is
returned on each call
- * because the Snowball stemmers are stateful and not thread-safe.
+ * {@return a thread-safe factory for this language's Snowball stemmer} The
factory may be
+ * shared; each {@linkplain StemmerFactory#newStemmer() minted stemmer} is
confined to the
+ * calling thread.
+ */
+ public StemmerFactory stemmerFactory() {
+ return new SnowballStemmerFactory(stemmerAlgorithm);
+ }
+
+ /**
+ * {@return a new {@link Stemmer} for this language} The returned {@link
SnowballStemmer} is
+ * thread-safe and may be shared across threads.
*/
public Stemmer newStemmer() {
return new SnowballStemmer(stemmerAlgorithm);
@@ -70,8 +81,10 @@ public record NormalizationProfile(String language,
SnowballStemmer.ALGORITHM st
/**
* Returns a matching analyzer for this language: NFC, case folding, the
language's
- * {@linkplain #accentFold() diacritic fold} when it has one, then stemming.
Each call builds an
- * independent analyzer with its own stemmer, so use one per thread when
stemming.
+ * {@linkplain #accentFold() diacritic fold} when it has one, then stemming.
The returned
+ * analyzer is thread-safe, and repeated words resolve from a bounded
per-thread stem cache
+ * instead of being re-stemmed. The cache is keyed to the thread, so build
the analyzer once
+ * and reuse it from threads that live across many calls, such as a fixed
platform-thread pool.
*
* @return the analyzer.
*/
@@ -80,6 +93,6 @@ public record NormalizationProfile(String language,
SnowballStemmer.ALGORITHM st
if (accentFold != null) {
builder.transform(Dimension.ACCENT_FOLD, accentFold);
}
- return builder.stem(newStemmer()).build();
+ return builder.stem(stemmerFactory()).build();
}
}
diff --git
a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/TermAnalyzer.java
b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/TermAnalyzer.java
index eda2a032e..00e8d1851 100644
---
a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/TermAnalyzer.java
+++
b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/TermAnalyzer.java
@@ -25,7 +25,9 @@ import java.util.Locale;
import java.util.Set;
import opennlp.tools.lemmatizer.Lemmatizer;
+import opennlp.tools.stemmer.CachingStemmer;
import opennlp.tools.stemmer.Stemmer;
+import opennlp.tools.stemmer.StemmerFactory;
import opennlp.tools.tokenize.uax29.WordTokenizer;
import opennlp.tools.util.Span;
@@ -41,10 +43,12 @@ import opennlp.tools.util.Span;
* {@link Dimension#LEMMA} are enabled by supplying a {@link Stemmer} or
{@link Lemmatizer}.</p>
*
* <p>An instance is immutable and is thread-safe when its configured
transforms are. The built-in
- * character normalizers are stateless, but the Snowball stemmers are not, so
an analyzer configured
- * with a {@link Stemmer} (for example through {@code
NormalizationProfile.matchingAnalyzer()}) should
- * not be shared across threads when {@link Dimension#STEM} is used. Build one
with
- * {@link #builder()}.</p>
+ * character normalizers are stateless and the Snowball stemmers are
thread-safe. When
+ * {@link Dimension#STEM} is enabled through {@link
Builder#stem(StemmerFactory)}, stemming is
+ * routed through a {@link CachingStemmer}: the analyzer is safe to share
across threads even when
+ * the factory mints stateful stemmers, and repeated words resolve from a
bounded per-thread cache
+ * instead of being re-stemmed. A raw {@link Stemmer} passed to {@link
Builder#stem(Stemmer)} is
+ * only safe when that stemmer is itself thread-safe or the analyzer is
confined to one thread.</p>
*/
public final class TermAnalyzer {
@@ -448,6 +452,37 @@ public final class TermAnalyzer {
return this;
}
+ /**
+ * Enables {@link Dimension#STEM} through a {@link StemmerFactory}. The
analyzer receives a
+ * {@link CachingStemmer} with the {@linkplain
CachingStemmer#DEFAULT_CAPACITY default cache
+ * capacity}: it can be shared across threads, and repeated words resolve
from a bounded
+ * per-thread cache instead of being re-stemmed. The cache is keyed to the
thread, so the
+ * memoization pays off on threads reused across many tokens (a fixed
platform-thread pool);
+ * on a virtual-thread-per-task executor every task starts with an empty
cache.
+ *
+ * @param factory The stemmer factory. Must not be {@code null}.
+ * @return this builder
+ * @throws IllegalArgumentException if {@code factory} is {@code null}.
+ */
+ public Builder stem(StemmerFactory factory) {
+ return stem(factory, CachingStemmer.DEFAULT_CAPACITY);
+ }
+
+ /**
+ * Enables {@link Dimension#STEM} through a {@link StemmerFactory} with an
explicit stem cache
+ * capacity, otherwise identical to {@link #stem(StemmerFactory)}.
+ *
+ * @param factory The stemmer factory. Must not be {@code null}.
+ * @param cacheSize The maximum number of word-to-stem entries kept per
thread; must be
+ * positive.
+ * @return this builder
+ * @throws IllegalArgumentException if {@code factory} is {@code null} or
{@code cacheSize} is
+ * not positive.
+ */
+ public Builder stem(StemmerFactory factory, int cacheSize) {
+ return stem(new CachingStemmer(factory, cacheSize));
+ }
+
/**
* Enables {@link Dimension#LEMMA} through the given lemmatizer.
*
diff --git
a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/CachingStemmerTest.java
b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/CachingStemmerTest.java
new file mode 100644
index 000000000..6890dbca8
--- /dev/null
+++
b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/CachingStemmerTest.java
@@ -0,0 +1,139 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.stemmer;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+import opennlp.tools.stemmer.snowball.SnowballStemmer;
+import opennlp.tools.stemmer.snowball.SnowballStemmerFactory;
+
+class CachingStemmerTest {
+
+ private static final SnowballStemmerFactory ENGLISH =
+ new SnowballStemmerFactory(SnowballStemmer.ALGORITHM.ENGLISH);
+
+ @Test
+ void cachedResultsMatchUncachedResults() {
+ Stemmer reference = ENGLISH.newStemmer();
+ CachingStemmer cached = new CachingStemmer(ENGLISH);
+
+ List<String> words = List.of("running", "declining", "conspiracies",
"this",
+ "running", "declining", "ab'yle", "running");
+ for (String word : words) {
+ Assertions.assertEquals(reference.stem(word).toString(),
cached.stem(word).toString(),
+ "mismatch for '" + word + "'");
+ }
+ }
+
+ @Test
+ void repeatedWordsHitTheCache() {
+ AtomicInteger delegateCalls = new AtomicInteger();
+ StemmerFactory countingFactory = () -> {
+ Stemmer delegate = ENGLISH.newStemmer();
+ return word -> {
+ delegateCalls.incrementAndGet();
+ return delegate.stem(word);
+ };
+ };
+
+ CachingStemmer cached = new CachingStemmer(countingFactory);
+ for (int i = 0; i < 100; i++) {
+ Assertions.assertEquals("run", cached.stem("running").toString());
+ }
+ Assertions.assertEquals(1, delegateCalls.get());
+ }
+
+ @Test
+ void evictionKeepsResultsCorrect() {
+ AtomicInteger delegateCalls = new AtomicInteger();
+ StemmerFactory countingFactory = () -> {
+ Stemmer delegate = ENGLISH.newStemmer();
+ return word -> {
+ delegateCalls.incrementAndGet();
+ return delegate.stem(word);
+ };
+ };
+
+ // Capacity 2 with a 3-word cycle: every access evicts the word needed two
steps later.
+ CachingStemmer cached = new CachingStemmer(countingFactory, 2);
+ Stemmer reference = ENGLISH.newStemmer();
+ List<String> words = List.of("running", "declining", "conspiracies");
+ for (int round = 0; round < 5; round++) {
+ for (String word : words) {
+ Assertions.assertEquals(reference.stem(word).toString(),
cached.stem(word).toString());
+ }
+ }
+ Assertions.assertTrue(delegateCalls.get() > 3,
+ "expected evictions to force repeated delegate calls, got " +
delegateCalls.get());
+ }
+
+ @Test
+ void stemAllBypassesTheCache() {
+ CachingStemmer cached = new CachingStemmer(ENGLISH);
+ List<CharSequence> forms = cached.stemAll("running");
+ Assertions.assertEquals(1, forms.size());
+ Assertions.assertEquals("run", forms.getFirst().toString());
+ }
+
+ @Test
+ void isSafeUnderConcurrentUse() throws Exception {
+ Stemmer reference = ENGLISH.newStemmer();
+ List<String> words = List.of("running", "declining", "conspiracies",
"annotations",
+ "photographers", "responsibilities", "this", "querying");
+ List<String> expected = new ArrayList<>(words.size());
+ for (String word : words) {
+ expected.add(reference.stem(word).toString());
+ }
+
+ // Capacity below the vocabulary size, so threads continuously evict
within their own cache.
+ CachingStemmer cached = new CachingStemmer(ENGLISH, 4);
+ try (ExecutorService pool = Executors.newFixedThreadPool(8)) {
+ Future<?>[] tasks = new Future<?>[32];
+ for (int i = 0; i < tasks.length; i++) {
+ final int offset = i;
+ tasks[i] = pool.submit(() -> {
+ for (int n = 0; n < 200; n++) {
+ for (int w = 0; w < words.size(); w++) {
+ int idx = (w + offset) % words.size();
+ Assertions.assertEquals(expected.get(idx),
cached.stem(words.get(idx)).toString());
+ }
+ }
+ });
+ }
+ for (Future<?> task : tasks) {
+ task.get(30, TimeUnit.SECONDS);
+ }
+ }
+ }
+
+ @Test
+ void rejectsInvalidArguments() {
+ Assertions.assertThrows(IllegalArgumentException.class, () -> new
CachingStemmer(null));
+ Assertions.assertThrows(IllegalArgumentException.class, () -> new
CachingStemmer(ENGLISH, 0));
+ Assertions.assertThrows(IllegalArgumentException.class, () -> new
CachingStemmer(ENGLISH, -1));
+ }
+}
diff --git
a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/StemmerFactoryTest.java
b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/StemmerFactoryTest.java
new file mode 100644
index 000000000..9a19d0110
--- /dev/null
+++
b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/StemmerFactoryTest.java
@@ -0,0 +1,376 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.stemmer;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.EnumSource;
+
+import opennlp.tools.stemmer.snowball.SnowballStemmer;
+import opennlp.tools.stemmer.snowball.SnowballStemmerFactory;
+
+class StemmerFactoryTest {
+
+ @Test
+ void snowballFactoryMatchesDirectStemmer() {
+ StemmerFactory factory = new
SnowballStemmerFactory(SnowballStemmer.ALGORITHM.ENGLISH);
+ Stemmer direct = new SnowballStemmer(SnowballStemmer.ALGORITHM.ENGLISH);
+ Assertions.assertEquals(direct.stem("running"),
factory.newStemmer().stem("running"));
+ }
+
+ @Test
+ void porterFactoryMatchesDirectStemmer() {
+ StemmerFactory factory = new PorterStemmerFactory();
+ Stemmer direct = new PorterStemmer();
+ Assertions.assertEquals(direct.stem("running"),
factory.newStemmer().stem("running"));
+ }
+
+ @Test
+ void stemAllDefaultReturnsSingleForm() {
+ StemmerFactory factory = new
SnowballStemmerFactory(SnowballStemmer.ALGORITHM.ENGLISH);
+ List<CharSequence> forms = factory.newStemmer().stemAll("running");
+ Assertions.assertEquals(1, forms.size());
+ Assertions.assertEquals("run", forms.getFirst());
+ }
+
+ @Test
+ void factoryRejectsInvalidRepeat() {
+ Assertions.assertThrows(IllegalArgumentException.class,
+ () -> new SnowballStemmerFactory(SnowballStemmer.ALGORITHM.ENGLISH,
0));
+ }
+
+ @Test
+ void sharingStemmerMatchesSingleThreadReference() throws Exception {
+ StemmerFactory factory = new
SnowballStemmerFactory(SnowballStemmer.ALGORITHM.ENGLISH);
+ Stemmer reference = factory.newStemmer();
+ SharingStemmer shared = new SharingStemmer(factory);
+
+ Assertions.assertEquals(reference.stem("running"), shared.stem("running"));
+ Assertions.assertEquals(reference.stem("this"), shared.stem("this"));
+ }
+
+ @Test
+ void sharingStemmerIsSafeUnderConcurrentUse() throws Exception {
+ StemmerFactory factory = new
SnowballStemmerFactory(SnowballStemmer.ALGORITHM.ENGLISH);
+ SharingStemmer shared = new SharingStemmer(factory);
+ CharSequence expectedRunning = factory.newStemmer().stem("running");
+ CharSequence expectedDeclining = factory.newStemmer().stem("declining");
+
+ try (ExecutorService pool = Executors.newFixedThreadPool(8)) {
+ Future<?>[] tasks = new Future<?>[64];
+ for (int i = 0; i < tasks.length; i++) {
+ tasks[i] = pool.submit(() -> {
+ for (int n = 0; n < 200; n++) {
+ Assertions.assertEquals(expectedRunning, shared.stem("running"));
+ Assertions.assertEquals(expectedDeclining,
shared.stem("declining"));
+ }
+ });
+ }
+ for (Future<?> task : tasks) {
+ task.get(30, TimeUnit.SECONDS);
+ }
+ }
+ }
+
+ @Test
+ void snowballStemmerIsSafeUnderConcurrentUse() throws Exception {
+ Stemmer shared = new SnowballStemmer(SnowballStemmer.ALGORITHM.ENGLISH);
+ CharSequence expectedRunning = shared.stem("running");
+ CharSequence expectedDeclining = shared.stem("declining");
+
+ try (ExecutorService pool = Executors.newFixedThreadPool(8)) {
+ Future<?>[] tasks = new Future<?>[64];
+ for (int i = 0; i < tasks.length; i++) {
+ tasks[i] = pool.submit(() -> {
+ for (int n = 0; n < 200; n++) {
+ Assertions.assertEquals(expectedRunning, shared.stem("running"));
+ Assertions.assertEquals(expectedDeclining,
shared.stem("declining"));
+ }
+ });
+ }
+ for (Future<?> task : tasks) {
+ task.get(30, TimeUnit.SECONDS);
+ }
+ }
+ }
+
+ @Test
+ void sharingStemmerRejectsNullFactory() {
+ Assertions.assertThrows(IllegalArgumentException.class, () -> new
SharingStemmer(null));
+ }
+
+ @ParameterizedTest
+ @EnumSource(SnowballStemmer.ALGORITHM.class)
+ void everyAlgorithmIsSafeUnderConcurrentUse(SnowballStemmer.ALGORITHM
algorithm)
+ throws Exception {
+ // Words across scripts; stemming any input is deterministic, so a
mismatch under
+ // concurrency signals corrupted shared state, regardless of the input
language.
+ List<String> words = List.of("running", "declining", "ab'yle", "prévoyant",
+ "επιστροφή", "яблоками", "استفتياكما", "skrøbeligheder");
+ Stemmer reference = new SnowballStemmer(algorithm);
+ List<String> expected = new ArrayList<>(words.size());
+ for (String word : words) {
+ expected.add(reference.stem(word).toString());
+ }
+
+ Stemmer shared = new SnowballStemmer(algorithm);
+ try (ExecutorService pool = Executors.newFixedThreadPool(4)) {
+ Future<?>[] tasks = new Future<?>[16];
+ for (int i = 0; i < tasks.length; i++) {
+ final int offset = i;
+ tasks[i] = pool.submit(() -> {
+ for (int n = 0; n < 50; n++) {
+ for (int w = 0; w < words.size(); w++) {
+ int idx = (w + offset) % words.size();
+ Assertions.assertEquals(expected.get(idx),
shared.stem(words.get(idx)).toString());
+ }
+ }
+ });
+ }
+ for (Future<?> task : tasks) {
+ task.get(30, TimeUnit.SECONDS);
+ }
+ }
+ }
+
+ @Test
+ void ownerThreadCanStemConcurrentlyWithOtherThreads() throws Exception {
+ // The calling thread claims the owner fast path first; other threads must
transition the
+ // stemmer to per-thread state while the owner keeps stemming.
+ Stemmer shared = new SnowballStemmer(SnowballStemmer.ALGORITHM.ENGLISH);
+ CharSequence expectedRunning = shared.stem("running");
+ CharSequence expectedDeclining = shared.stem("declining");
+
+ CountDownLatch started = new CountDownLatch(4);
+ try (ExecutorService pool = Executors.newFixedThreadPool(4)) {
+ Future<?>[] tasks = new Future<?>[4];
+ for (int i = 0; i < tasks.length; i++) {
+ tasks[i] = pool.submit(() -> {
+ started.countDown();
+ for (int n = 0; n < 500; n++) {
+ Assertions.assertEquals(expectedRunning, shared.stem("running"));
+ Assertions.assertEquals(expectedDeclining,
shared.stem("declining"));
+ }
+ });
+ }
+ Assertions.assertTrue(started.await(10, TimeUnit.SECONDS));
+ // Owner thread stems concurrently with the pool threads.
+ for (int n = 0; n < 500; n++) {
+ Assertions.assertEquals(expectedRunning, shared.stem("running"));
+ Assertions.assertEquals(expectedDeclining, shared.stem("declining"));
+ }
+ for (Future<?> task : tasks) {
+ task.get(30, TimeUnit.SECONDS);
+ }
+ }
+ }
+
+ @Test
+ void snowballStemmerIsSafeOnVirtualThreads() throws Exception {
+ // Every task runs on a fresh virtual thread, exercising per-thread state
creation on each
+ // access rather than reuse from a fixed pool.
+ Stemmer shared = new SnowballStemmer(SnowballStemmer.ALGORITHM.ENGLISH);
+ CharSequence expected = shared.stem("running");
+
+ try (ExecutorService pool = Executors.newVirtualThreadPerTaskExecutor()) {
+ List<Future<?>> tasks = new ArrayList<>(256);
+ for (int i = 0; i < 256; i++) {
+ tasks.add(pool.submit(() -> {
+ for (int n = 0; n < 20; n++) {
+ Assertions.assertEquals(expected, shared.stem("running"));
+ }
+ }));
+ }
+ for (Future<?> task : tasks) {
+ task.get(30, TimeUnit.SECONDS);
+ }
+ }
+ }
+
+ @Test
+ void sharingPorterStemmerIsSafeUnderConcurrentUse() throws Exception {
+ // PorterStemmer is the remaining stateful engine; SharingStemmer must
isolate it per thread.
+ Stemmer reference = new PorterStemmer();
+ CharSequence expectedRunning = reference.stem("running");
+ CharSequence expectedConspiracies = reference.stem("conspiracies");
+ Stemmer shared = new SharingStemmer(new PorterStemmerFactory());
+
+ try (ExecutorService pool = Executors.newFixedThreadPool(8)) {
+ Future<?>[] tasks = new Future<?>[64];
+ for (int i = 0; i < tasks.length; i++) {
+ tasks[i] = pool.submit(() -> {
+ for (int n = 0; n < 200; n++) {
+ Assertions.assertEquals(expectedRunning, shared.stem("running"));
+ Assertions.assertEquals(expectedConspiracies,
shared.stem("conspiracies"));
+ }
+ });
+ }
+ for (Future<?> task : tasks) {
+ task.get(30, TimeUnit.SECONDS);
+ }
+ }
+ }
+ @Test
+ void factoryPassesRepeatThrough() {
+ // "internationalization" stems differently at repeat 1 and 2, witnessing
the pass-through.
+ final Stemmer once = new
SnowballStemmerFactory(SnowballStemmer.ALGORITHM.ENGLISH, 1).newStemmer();
+ final Stemmer twice = new
SnowballStemmerFactory(SnowballStemmer.ALGORITHM.ENGLISH, 2).newStemmer();
+ Assertions.assertEquals("internation",
once.stem("internationalization").toString());
+ Assertions.assertEquals("intern",
twice.stem("internationalization").toString());
+ Assertions.assertEquals(
+ new SnowballStemmer(SnowballStemmer.ALGORITHM.ENGLISH,
2).stem("internationalization").toString(),
+ twice.stem("internationalization").toString());
+ }
+
+ @Test
+ void factoryRejectsNullAlgorithmWithIllegalArgument() {
+ Assertions.assertThrows(IllegalArgumentException.class,
+ () -> new SnowballStemmerFactory(null, 1));
+ }
+
+ @Test
+ void snowballConstructorValidatesLikeTheFactory() {
+ Assertions.assertThrows(IllegalArgumentException.class,
+ () -> new SnowballStemmer(null, 1));
+ Assertions.assertThrows(IllegalArgumentException.class,
+ () -> new SnowballStemmer(SnowballStemmer.ALGORITHM.ENGLISH, 0));
+ }
+
+ @Test
+ void sharingStemmerForwardsStemAllToMultiOutputDelegate() {
+ // A multi-output delegate must keep its full result list through the
wrapper.
+ final StemmerFactory multiOutput = () -> new Stemmer() {
+ @Override
+ public CharSequence stem(CharSequence word) {
+ return word.toString() + "-a";
+ }
+
+ @Override
+ public List<CharSequence> stemAll(CharSequence word) {
+ return List.of(word.toString() + "-a", word.toString() + "-b");
+ }
+ };
+ final SharingStemmer sharing = new SharingStemmer(multiOutput);
+ Assertions.assertEquals(2, sharing.stemAll("run").size());
+ Assertions.assertEquals("run-b", sharing.stemAll("run").get(1).toString());
+ }
+
+ @Test
+ void clearThreadLocalStateEmptiesTheOwnerCache() {
+ // On the owning thread the state object is retained and reset: the cache
must be empty
+ // afterwards, observable as a fresh delegate call for a previously cached
word.
+ final List<String> delegateCalls = new ArrayList<>();
+ final StemmerFactory counting = () -> word -> {
+ delegateCalls.add(word.toString());
+ return word.toString() + "-s";
+ };
+ final CachingStemmer caching = new CachingStemmer(counting);
+ caching.stem("dog");
+ caching.stem("dog");
+ Assertions.assertEquals(1, delegateCalls.size(), "second lookup is served
from the cache");
+ caching.clearThreadLocalState();
+ Assertions.assertEquals("dog-s", caching.stem("dog").toString());
+ Assertions.assertEquals(2, delegateCalls.size(), "the clear emptied the
cache");
+ }
+
+ @Test
+ void clearThreadLocalStateReleasesAWorkerThreadsDelegate() throws Exception {
+ // On a non-owner thread the per-thread delegate is removed and rebuilt on
next use.
+ final List<Integer> built = new ArrayList<>();
+ final StemmerFactory counting = () -> {
+ synchronized (built) {
+ built.add(built.size());
+ }
+ return word -> word.toString() + "-s";
+ };
+ final SharingStemmer sharing = new SharingStemmer(counting);
+ sharing.stem("owner");
+ final int builtForOwner = built.size();
+ final Thread worker = new Thread(() -> {
+ sharing.stem("dog");
+ sharing.stem("cat");
+ final int beforeClear = built.size();
+ sharing.clearThreadLocalState();
+ sharing.stem("fish");
+ Assertions.assertEquals(beforeClear + 1, built.size(),
+ "a fresh delegate is built after the clear");
+ });
+ worker.start();
+ worker.join(30_000);
+ Assertions.assertTrue(built.size() > builtForOwner, "the worker built its
own delegate");
+
+ final SnowballStemmer snowball = new
SnowballStemmer(SnowballStemmer.ALGORITHM.ENGLISH);
+ Assertions.assertEquals("run", snowball.stem("running").toString());
+ snowball.clearThreadLocalState();
+ Assertions.assertEquals("run", snowball.stem("running").toString());
+ }
+
+ @Test
+ void nullWordThrowsIllegalArgumentAcrossTheStemmerApi() {
+ final SnowballStemmerFactory factory =
+ new SnowballStemmerFactory(SnowballStemmer.ALGORITHM.ENGLISH);
+ Assertions.assertThrows(IllegalArgumentException.class,
+ () -> new
SnowballStemmer(SnowballStemmer.ALGORITHM.ENGLISH).stem(null));
+ Assertions.assertThrows(IllegalArgumentException.class,
+ () -> factory.newStemmer().stem(null));
+ Assertions.assertThrows(IllegalArgumentException.class,
+ () -> new CachingStemmer(factory).stem(null));
+ Assertions.assertThrows(IllegalArgumentException.class,
+ () -> new SharingStemmer(factory).stem(null));
+ Assertions.assertThrows(IllegalArgumentException.class,
+ () -> new PorterStemmer().stem((CharSequence) null));
+ }
+
+ @Test
+ void hugeCapacityConstructsAndStems() {
+ // Pins the overflow-safe table sizing: capacities near Integer.MAX_VALUE
are documented
+ // as legal ("must be positive") and must not wrap the LinkedHashMap
initial capacity.
+ final CachingStemmer cached = new CachingStemmer(
+ new SnowballStemmerFactory(SnowballStemmer.ALGORITHM.ENGLISH),
Integer.MAX_VALUE);
+ Assertions.assertEquals("run", cached.stem("running").toString());
+ }
+
+ @Test
+ void clearCacheForcesRestemmingOnTheCallingThread() {
+ final AtomicInteger delegateCalls = new AtomicInteger();
+ final StemmerFactory counting = () -> word -> {
+ delegateCalls.incrementAndGet();
+ return word.toString();
+ };
+ final CachingStemmer cached = new CachingStemmer(counting);
+
+ cached.stem("running");
+ cached.stem("running");
+ Assertions.assertEquals(1, delegateCalls.get(), "the repeat is served from
the cache");
+
+ cached.clearCache();
+ cached.stem("running");
+ Assertions.assertEquals(2, delegateCalls.get(),
+ "after clearCache() the word goes through the delegate again");
+ }
+}
diff --git
a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/StemmerFactoryUsageExampleTest.java
b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/StemmerFactoryUsageExampleTest.java
new file mode 100644
index 000000000..c0b97941d
--- /dev/null
+++
b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/StemmerFactoryUsageExampleTest.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.stemmer;
+
+import java.util.List;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+import opennlp.tools.stemmer.snowball.SnowballStemmer;
+import opennlp.tools.stemmer.snowball.SnowballStemmerFactory;
+
+/**
+ * Runs the manual's stemmer factory examples (docbkx {@code stemmer.xml})
verbatim: every
+ * value the chapter states is asserted here, so a change breaking this test
breaks the
+ * manual.
+ */
+public class StemmerFactoryUsageExampleTest {
+
+ /**
+ * Factory to {@code newStemmer()} to a single stem and the default {@code
stemAll} list.
+ */
+ @Test
+ void testSnowballFactoryStemsOneWord() {
+ final StemmerFactory factory =
+ new SnowballStemmerFactory(SnowballStemmer.ALGORITHM.ENGLISH);
+ final Stemmer stemmer = factory.newStemmer();
+
+ Assertions.assertEquals("run", stemmer.stem("running").toString());
+ Assertions.assertEquals(List.of("run"),
+
stemmer.stemAll("running").stream().map(CharSequence::toString).toList());
+ }
+
+ /**
+ * {@link SharingStemmer} and {@link CachingStemmer} produce the same stem
as a fresh
+ * factory stemmer for the same word.
+ */
+ @Test
+ void testSharingAndCachingStemmersMatchFactoryStem() {
+ final StemmerFactory factory =
+ new SnowballStemmerFactory(SnowballStemmer.ALGORITHM.ENGLISH);
+ final CharSequence expected = factory.newStemmer().stem("running");
+
+ Assertions.assertEquals(expected, new
SharingStemmer(factory).stem("running"));
+ Assertions.assertEquals(expected, new
CachingStemmer(factory).stem("running"));
+ }
+}
diff --git
a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/NormalizationProfilesTest.java
b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/NormalizationProfilesTest.java
index ba39a5cf1..149139847 100644
---
a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/NormalizationProfilesTest.java
+++
b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/NormalizationProfilesTest.java
@@ -19,6 +19,9 @@ package opennlp.tools.util.normalizer;
import java.util.EnumSet;
import java.util.List;
import java.util.Set;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
import java.util.stream.Collectors;
import org.junit.jupiter.api.Test;
@@ -201,4 +204,24 @@ public class NormalizationProfilesTest {
assertThrows(IllegalArgumentException.class, () ->
NormalizationProfiles.detect(null, detector));
assertThrows(IllegalArgumentException.class, () ->
NormalizationProfiles.detect("text", null));
}
+
+ @Test
+ void testMatchingAnalyzerIsThreadSafeForStemming() throws Exception {
+ final TermAnalyzer analyzer =
NormalizationProfiles.forLanguage("eng").orElseThrow().matchingAnalyzer();
+ final String expected =
analyzer.analyze("running").getFirst().at(Dimension.STEM);
+
+ try (ExecutorService pool = Executors.newFixedThreadPool(8)) {
+ final Future<?>[] tasks = new Future<?>[32];
+ for (int i = 0; i < tasks.length; i++) {
+ tasks[i] = pool.submit(() -> {
+ for (int n = 0; n < 50; n++) {
+ assertEquals(expected,
analyzer.analyze("running").getFirst().at(Dimension.STEM));
+ }
+ });
+ }
+ for (final Future<?> task : tasks) {
+ task.get(30, java.util.concurrent.TimeUnit.SECONDS);
+ }
+ }
+ }
}
diff --git
a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/TermAnalyzerTest.java
b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/TermAnalyzerTest.java
index db9018e99..39232e16e 100644
---
a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/TermAnalyzerTest.java
+++
b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/TermAnalyzerTest.java
@@ -30,6 +30,10 @@ import org.junit.jupiter.api.Test;
import opennlp.tools.lemmatizer.Lemmatizer;
import opennlp.tools.stemmer.PorterStemmer;
+import opennlp.tools.stemmer.Stemmer;
+import opennlp.tools.stemmer.StemmerFactory;
+import opennlp.tools.stemmer.snowball.SnowballStemmer;
+import opennlp.tools.stemmer.snowball.SnowballStemmerFactory;
import opennlp.tools.util.Span;
import static org.junit.jupiter.api.Assertions.assertEquals;
@@ -318,11 +322,33 @@ public class TermAnalyzerTest {
.transform(null, CaseFoldCharSequenceNormalizer.getInstance()));
assertThrows(IllegalArgumentException.class,
() -> TermAnalyzer.builder().transform(Dimension.CASE_FOLD, null));
- assertThrows(IllegalArgumentException.class, () ->
TermAnalyzer.builder().stem(null));
+ assertThrows(IllegalArgumentException.class,
+ () -> TermAnalyzer.builder().stem((Stemmer) null));
+ assertThrows(IllegalArgumentException.class,
+ () -> TermAnalyzer.builder().stem((StemmerFactory) null));
+ assertThrows(IllegalArgumentException.class,
+ () -> TermAnalyzer.builder().stem(new SnowballStemmerFactory(
+ SnowballStemmer.ALGORITHM.ENGLISH), 0));
assertThrows(IllegalArgumentException.class, () ->
TermAnalyzer.builder().lemmatize(null));
assertThrows(IllegalArgumentException.class, () ->
TermAnalyzer.builder().tokenizer(null));
}
+ @Test
+ void testStemFactoryCachedResultsMatchDirectStemmer() {
+ final TermAnalyzer cached = TermAnalyzer.builder().caseFold()
+ .stem(new
SnowballStemmerFactory(SnowballStemmer.ALGORITHM.ENGLISH)).build();
+ final TermAnalyzer direct = TermAnalyzer.builder().caseFold()
+ .stem(new SnowballStemmer(SnowballStemmer.ALGORITHM.ENGLISH)).build();
+
+ final String text = "Running runners kept running while the RUNNING
continued";
+ final List<Term> cachedTerms = cached.analyze(text);
+ final List<Term> directTerms = direct.analyze(text);
+ assertEquals(directTerms.size(), cachedTerms.size());
+ for (int i = 0; i < directTerms.size(); i++) {
+ assertEquals(directTerms.get(i).at(Dimension.STEM),
cachedTerms.get(i).at(Dimension.STEM));
+ }
+ }
+
@Test
void testMaxTokenLengthRejectsNonPositiveValues() {
assertThrows(IllegalArgumentException.class, () ->
TermAnalyzer.builder().maxTokenLength(0));
diff --git a/opennlp-docs/src/docbkx/opennlp.xml
b/opennlp-docs/src/docbkx/opennlp.xml
index 263c5f96c..36641c2c8 100644
--- a/opennlp-docs/src/docbkx/opennlp.xml
+++ b/opennlp-docs/src/docbkx/opennlp.xml
@@ -108,6 +108,7 @@ under the License.
<xi:include xmlns:xi="http://www.w3.org/2001/XInclude";
href="./sentiment.xml" />
<xi:include xmlns:xi="http://www.w3.org/2001/XInclude";
href="./postagger.xml" />
<xi:include xmlns:xi="http://www.w3.org/2001/XInclude";
href="./lemmatizer.xml" />
+ <xi:include xmlns:xi="http://www.w3.org/2001/XInclude";
href="./stemmer.xml" />
<xi:include xmlns:xi="http://www.w3.org/2001/XInclude";
href="./spellcheck.xml" />
<xi:include xmlns:xi="http://www.w3.org/2001/XInclude";
href="./chunker.xml" />
<xi:include xmlns:xi="http://www.w3.org/2001/XInclude";
href="./parser.xml" />
diff --git a/opennlp-docs/src/docbkx/stemmer.xml
b/opennlp-docs/src/docbkx/stemmer.xml
new file mode 100644
index 000000000..248b310c1
--- /dev/null
+++ b/opennlp-docs/src/docbkx/stemmer.xml
@@ -0,0 +1,72 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE chapter PUBLIC "-//OASIS//DTD DocBook XML 5.0//EN"
+"http://docbook.org/xml/5.0/dtd/docbook.dtd";[
+]>
+<!-- Licensed to the Apache Software Foundation (ASF) under one or more
contributor
+ license agreements. See the NOTICE file distributed with this work for
additional
+ information regarding copyright ownership. The ASF licenses this file to
+ you under the Apache License, Version 2.0 (the "License"); you may not
use
+ this file except in compliance with the License. You may obtain a copy
of
+ the License at http://www.apache.org/licenses/LICENSE-2.0 Unless
required
+ by applicable law or agreed to in writing, software distributed under
the
+ License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
CONDITIONS
+ OF ANY KIND, either express or implied. See the License for the specific
+ language governing permissions and limitations under the License. -->
+
+<chapter xml:id="tools.stemmer">
+
+ <title>Stemmer</title>
+
+ <section xml:id="tools.stemmer.introduction">
+ <title>Introduction</title>
+ <para>
+ A stemmer reduces a word form to a shorter root.
OpenNLP ships rule-based
+ engines such as Snowball and Porter through the
+ <code>opennlp.tools.stemmer.Stemmer</code> interface.
Stemming is
+ language-specific and lossy: it is useful for search
and feature reduction,
+ not as a substitute for lemmatization when a dictionary
form is required.
+ </para>
+ </section>
+
+ <section xml:id="tools.stemmer.factory">
+ <title>StemmerFactory</title>
+ <para>
+ <code>StemmerFactory</code> captures a stemmer
configuration once and mints
+ configured <code>Stemmer</code> instances on demand. It
is a plain supplier
+ of stemmers, not a <code>BaseToolFactory</code> loaded
from a model
+ manifest. The English Snowball factory needs no
external data:
+ <code>StemmerFactoryUsageExampleTest</code> asserts the
behavior shown
+ here.
+ <programlisting language="java"><![CDATA[
+StemmerFactory factory = new
SnowballStemmerFactory(SnowballStemmer.ALGORITHM.ENGLISH);
+Stemmer stemmer = factory.newStemmer();
+
+stemmer.stem("running"); // "run"
+stemmer.stemAll("running"); // ["run"]]]>
+ </programlisting>
+ </para>
+ </section>
+
+ <section xml:id="tools.stemmer.sharing">
+ <title>Sharing and caching</title>
+ <para>
+ Some stemmer implementations keep mutable engine state
and must not be
+ called from several threads at once.
<code>SharingStemmer</code> wraps a
+ <code>StemmerFactory</code> and routes each call to a
per-thread delegate,
+ so one instance can be shared safely.
<code>CachingStemmer</code> adds a
+ bounded per-thread LRU cache of word-to-stem mappings
on top of the same
+ factory pattern. Both wrappers return the same stem as
a fresh factory
+ stemmer for the same word:
+ <programlisting language="java"><![CDATA[
+StemmerFactory factory = new
SnowballStemmerFactory(SnowballStemmer.ALGORITHM.ENGLISH);
+
+new SharingStemmer(factory).stem("running"); // "run"
+new CachingStemmer(factory).stem("running"); // "run"]]>
+ </programlisting>
+ Thread-safe stemmers such as the shareable
<code>SnowballStemmer</code>
+ class do not need <code>SharingStemmer</code>.
Long-running containers
+ should call <code>clearThreadLocalState()</code> when a
pooled thread no
+ longer uses a sharing or caching stemmer.
+ </para>
+ </section>
+</chapter>
diff --git
a/opennlp-eval-tests/src/test/java/opennlp/tools/eval/MultiThreadedStemmerEval.java
b/opennlp-eval-tests/src/test/java/opennlp/tools/eval/MultiThreadedStemmerEval.java
new file mode 100644
index 000000000..aaac3c4c2
--- /dev/null
+++
b/opennlp-eval-tests/src/test/java/opennlp/tools/eval/MultiThreadedStemmerEval.java
@@ -0,0 +1,175 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.eval;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Random;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+import opennlp.tools.commons.ThreadSafe;
+import opennlp.tools.stemmer.PorterStemmerFactory;
+import opennlp.tools.stemmer.SharingStemmer;
+import opennlp.tools.stemmer.Stemmer;
+import opennlp.tools.stemmer.snowball.SnowballStemmer;
+import opennlp.tools.stemmer.snowball.SnowballStemmer.ALGORITHM;
+import opennlp.tools.util.normalizer.Dimension;
+import opennlp.tools.util.normalizer.NormalizationProfiles;
+import opennlp.tools.util.normalizer.Term;
+import opennlp.tools.util.normalizer.TermAnalyzer;
+
+/**
+ * Tests that the thread-safe stemmer implementations really are thread-safe
by hammering shared
+ * instances from many threads and comparing every result against a
single-threaded reference.
+ *
+ * @see ThreadSafe
+ * @see MultiThreadedToolsEval
+ */
+public class MultiThreadedStemmerEval extends AbstractEvalTest {
+
+ private static final int NUM_THREADS = 8;
+ private static final int RUNS_PER_THREAD = 500;
+
+ /**
+ * Words across scripts and languages, including apostrophes, diacritics,
Greek, Cyrillic and
+ * Arabic. Stemming is deterministic for any input, so every algorithm is
run over the whole
+ * list; a mismatch signals cross-thread corruption of the stemmer's
internal buffers.
+ */
+ private static final List<String> WORDS = List.of(
+ "running", "accompanying", "malediction", "softeners", "declining",
+ "importantíssimes", "besar", "accidentalment",
+ "abbattimento", "aborrecimentos", "importantísimas",
+ "esiintymispaikasta", "aabenbaringen", "skrøbeligheder",
"aftonringningen",
+ "buchbindergesellen", "mitverursacht", "prévoyant", "examinateurs",
+ "ab'yle", "kaçmamaktadır", "sarayı'nı",
+ "abbahagynám", "konstrukciójából", "vliegtuigtransport",
+ "absurdităţilor", "saracilor", "peledakan", "bhfeidhm",
+ "επιστροφή", "στρατιωτών", "красивая", "яблоками",
+ "استفتياكما", "استنتاجاتهما");
+
+ @Test
+ public void runSharedSnowballStemmersMultiThreaded() throws Exception {
+ for (ALGORITHM algorithm : ALGORITHM.values()) {
+ List<String> expected = referenceStems(new SnowballStemmer(algorithm));
+ hammer(new SnowballStemmer(algorithm), expected, "Snowball " +
algorithm);
+ }
+ }
+
+ @Test
+ public void runSharedPorterStemmerMultiThreaded() throws Exception {
+ SharingStemmer shared = new SharingStemmer(new PorterStemmerFactory());
+ List<String> expected = referenceStems(new
PorterStemmerFactory().newStemmer());
+ hammer(shared, expected, "SharingStemmer(Porter)");
+ }
+
+ @Test
+ public void runMatchingAnalyzersMultiThreaded() throws Exception {
+ for (String language : NormalizationProfiles.supportedLanguages()) {
+ TermAnalyzer analyzer =
+
NormalizationProfiles.forLanguage(language).orElseThrow().matchingAnalyzer();
+
+ List<String> expected = new ArrayList<>(WORDS.size());
+ for (String word : WORDS) {
+ expected.add(stemDimension(analyzer, word));
+ }
+
+ runInThreads(threadIndex -> {
+ List<Integer> order = shuffledIndexes(threadIndex);
+ for (int run = 0; run < RUNS_PER_THREAD / 10; run++) {
+ for (int i : order) {
+ Assertions.assertEquals(expected.get(i), stemDimension(analyzer,
WORDS.get(i)),
+ () -> "matchingAnalyzer(" + language + ") corrupted under
concurrency");
+ }
+ }
+ });
+ }
+ }
+
+ private static String stemDimension(TermAnalyzer analyzer, String word) {
+ List<Term> terms = analyzer.analyze(word);
+ if (terms.isEmpty()) {
+ return "";
+ }
+ StringBuilder sb = new StringBuilder();
+ for (Term term : terms) {
+ sb.append(term.at(Dimension.STEM)).append(' ');
+ }
+ return sb.toString();
+ }
+
+ private static List<String> referenceStems(Stemmer reference) {
+ List<String> expected = new ArrayList<>(WORDS.size());
+ for (String word : WORDS) {
+ expected.add(reference.stem(word).toString());
+ }
+ return expected;
+ }
+
+ private static void hammer(Stemmer shared, List<String> expected, String
label) throws Exception {
+ runInThreads(threadIndex -> {
+ List<Integer> order = shuffledIndexes(threadIndex);
+ for (int run = 0; run < RUNS_PER_THREAD; run++) {
+ for (int i : order) {
+ Assertions.assertEquals(expected.get(i),
shared.stem(WORDS.get(i)).toString(),
+ () -> label + " corrupted under concurrency for input '" +
WORDS.get(i) + "'");
+ }
+ }
+ });
+ }
+
+ /**
+ * Each thread visits the word list in its own order so that different
threads are (almost)
+ * always inside different words at the same time, maximizing the chance of
exposing shared
+ * mutable state.
+ */
+ private static List<Integer> shuffledIndexes(int seed) {
+ List<Integer> order = new ArrayList<>(WORDS.size());
+ for (int i = 0; i < WORDS.size(); i++) {
+ order.add(i);
+ }
+ Collections.shuffle(order, new Random(seed));
+ return order;
+ }
+
+ private interface ThreadBody {
+ void run(int threadIndex) throws Exception;
+ }
+
+ private static void runInThreads(ThreadBody body) throws Exception {
+ try (ExecutorService pool = Executors.newFixedThreadPool(NUM_THREADS)) {
+ List<Future<?>> tasks = new ArrayList<>(NUM_THREADS);
+ for (int t = 0; t < NUM_THREADS; t++) {
+ final int threadIndex = t;
+ tasks.add(pool.submit(() -> {
+ body.run(threadIndex);
+ return null;
+ }));
+ }
+ for (Future<?> task : tasks) {
+ task.get(120, TimeUnit.SECONDS);
+ }
+ }
+ }
+}
