rzo1 commented on code in PR #1163: URL: https://github.com/apache/opennlp/pull/1163#discussion_r3564735122
########## opennlp-api/src/main/java/opennlp/tools/stemmer/Stemmer.java: ########## @@ -4,7 +4,7 @@ * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at + * the License. You may obtain a copy of the License at Review Comment: Any reason for this whitespace change? ########## opennlp-api/src/main/java/opennlp/tools/stemmer/Stemmer.java: ########## @@ -17,10 +17,37 @@ package opennlp.tools.stemmer; +import java.util.List; + /** - * The stemmer is reducing a word to its stem. + * Reduces a word to its root form. + * + * <p>Thread safety is implementation-specific: check the implementation's documentation before Review Comment: I think it should be shorten to: "Thread safety is implementation specific.". The rest is generic text, which is not needed here IMHO. ########## opennlp-api/src/main/java/opennlp/tools/stemmer/StemmerFactory.java: ########## @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.stemmer; + +/** + * An immutable, thread-safe factory for {@link Stemmer} instances. + * + * <p>Stateful stemming engines hold per-call mutable buffers and must not be shared across Review Comment: This can be shorten per comment above. ########## opennlp-api/src/main/java/opennlp/tools/stemmer/StemmerFactory.java: ########## @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.stemmer; + +/** + * An immutable, thread-safe factory for {@link Stemmer} instances. + * + * <p>Stateful stemming engines hold per-call mutable buffers and must not be shared across + * threads. A {@code StemmerFactory} captures the configuration (algorithm, repeat count, + * dictionary path, ...) and mints a fresh {@link Stemmer} on each {@link #newStemmer()} call. The + * factory itself is safe to share; confine each returned {@link Stemmer} to a single thread, or + * route through a thread-local adapter when a component must be shared.</p> + * + * <p>Despite the name, this is not one of the {@code BaseToolFactory}-based tool factories (such + * as {@code LemmatizerFactory}) that are instantiated by name from a model manifest. It is a + * plain functional supplier of configured {@link Stemmer} instances and takes no part in the + * model-loading mechanism.</p> + * + * <p>Implementations must be immutable and thread-safe after construction.</p> + */ +public interface StemmerFactory { + + /** + * {@return a new {@link Stemmer} confined to the calling thread} Implementations must return a Review Comment: Since it is interface it should not talk about threading. ########## opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/SharingStemmer.java: ########## @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.stemmer; + +import java.util.List; + +import opennlp.tools.commons.ThreadSafe; +import opennlp.tools.util.OwnerOrPerThreadState; + +/** + * A {@link Stemmer} that is safe to share across threads by routing each call to a per-thread + * delegate minted from a {@link StemmerFactory}. Use it to share a stemmer whose implementation is + * <em>not</em> itself thread-safe, such as {@link PorterStemmer}; thread-safe implementations like + * {@code SnowballStemmer} do not need this wrapper. + * + * <p>The first thread uses a single owner delegate without touching {@link ThreadLocal} until a + * second thread appears, matching the {@link OwnerOrPerThreadState} pattern used by the + * thread-safe {@code *ME} components.</p> + */ +@ThreadSafe +public final class SharingStemmer implements Stemmer { Review Comment: Couldn't we just use generics to reduce the amount of this pattern in this PR? Since the pattern and code is mostly equal, this might be a way of dedup code, no? ########## opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/snowball/SnowballStemmerFactory.java: ########## @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.stemmer.snowball; + +import opennlp.tools.commons.ThreadSafe; +import opennlp.tools.stemmer.Stemmer; +import opennlp.tools.stemmer.StemmerFactory; + +/** + * A thread-safe factory that captures a Snowball stemmer configuration for APIs that accept a + * {@link StemmerFactory}. + * + * <p>{@link #newStemmer()} returns a thread-confined stemmer that drives its generated engine + * through a plain field, with none of the per-call thread routing of the shareable + * {@link SnowballStemmer}. This keeps the one-stemmer-per-thread pattern (and per-thread + * delegates inside caching or sharing wrappers) free of that indirection; use + * {@link SnowballStemmer} directly when a single shared instance is wanted instead.</p> + * + * @param algorithm The Snowball algorithm. Must not be {@code null}. + * @param repeat How many times to apply the stemmer per word; must be positive. + */ +@ThreadSafe +public record SnowballStemmerFactory(SnowballStemmer.ALGORITHM algorithm, int repeat) + implements StemmerFactory { + + /** + * Creates a factory with {@code repeat = 1}. + * + * @param algorithm The Snowball algorithm. Must not be {@code null}. + */ + public SnowballStemmerFactory(SnowballStemmer.ALGORITHM algorithm) { + this(algorithm, 1); + } + + /** + * Validates the components. + * + * @throws IllegalArgumentException Thrown if {@code algorithm} is {@code null} or + * {@code repeat} is not positive. + */ + public SnowballStemmerFactory { + if (algorithm == null) { + throw new IllegalArgumentException("algorithm must not be null"); + } + if (repeat <= 0) { + throw new IllegalArgumentException("repeat must be positive, got " + repeat); + } + } + + @Override + public Stemmer newStemmer() { + return new ConfinedStemmer(SnowballStemmer.engineFor(algorithm).get(), repeat); + } + + // One engine in a plain field: the zero-indirection product for thread-confined use, per the + // StemmerFactory contract. Not thread-safe. + private static final class ConfinedStemmer implements Stemmer { + + private final AbstractSnowballStemmer engine; + private final int repeat; + + private ConfinedStemmer(AbstractSnowballStemmer engine, int repeat) { + this.engine = engine; + this.repeat = repeat; + } + + @Override + public CharSequence stem(CharSequence word) { + engine.setCurrent(word.toString()); Review Comment: Does not honour interface contract (param must not be null) ########## opennlp-api/src/main/java/opennlp/tools/stemmer/Stemmer.java: ########## @@ -17,10 +17,37 @@ package opennlp.tools.stemmer; +import java.util.List; + /** - * The stemmer is reducing a word to its stem. + * Reduces a word to its root form. + * + * <p>Thread safety is implementation-specific: check the implementation's documentation before + * sharing an instance across threads. Stateful engines mutate internal buffers on each + * {@link #stem(CharSequence)} call; when an implementation is not documented as thread-safe, + * share a {@link StemmerFactory} across threads and confine each {@code Stemmer} to one thread, + * or wrap the factory in a thread-local adapter when a single {@code Stemmer} reference must be + * shared.</p> */ public interface Stemmer { + /** + * Stems {@code word}. + * + * @param word The input word. Must not be {@code null}. + * @return The stemmed form. + */ CharSequence stem(CharSequence word); + + /** + * {@return every stem form for {@code word}} The default returns a single-element list with Review Comment: ``` * {@return all stem forms for {@code word}} Defaults to a single-element list from * {@link #stem(CharSequence)}. Dictionary-based engines may override this to return * multiple roots; delegating wrappers must forward it to preserve the full list. ``` ########## opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/snowball/SnowballStemmerFactory.java: ########## @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.stemmer.snowball; + +import opennlp.tools.commons.ThreadSafe; +import opennlp.tools.stemmer.Stemmer; +import opennlp.tools.stemmer.StemmerFactory; + +/** + * A thread-safe factory that captures a Snowball stemmer configuration for APIs that accept a + * {@link StemmerFactory}. + * + * <p>{@link #newStemmer()} returns a thread-confined stemmer that drives its generated engine Review Comment: Needs to be more concise. This reads a bit like LLM-generated "bla". Maybe something like: Use this factory for the OpenNLP 2.x way of non thread safe stemmer usage or a like? wdyt? ########## opennlp-api/src/main/java/opennlp/tools/stemmer/Stemmer.java: ########## @@ -17,10 +17,37 @@ package opennlp.tools.stemmer; +import java.util.List; + /** - * The stemmer is reducing a word to its stem. + * Reduces a word to its root form. + * + * <p>Thread safety is implementation-specific: check the implementation's documentation before + * sharing an instance across threads. Stateful engines mutate internal buffers on each + * {@link #stem(CharSequence)} call; when an implementation is not documented as thread-safe, + * share a {@link StemmerFactory} across threads and confine each {@code Stemmer} to one thread, + * or wrap the factory in a thread-local adapter when a single {@code Stemmer} reference must be + * shared.</p> */ public interface Stemmer { + /** + * Stems {@code word}. + * + * @param word The input word. Must not be {@code null}. + * @return The stemmed form. + */ CharSequence stem(CharSequence word); + + /** + * {@return every stem form for {@code word}} The default returns a single-element list with + * {@link #stem(CharSequence)}. Dictionary-based engines that can produce several root forms for + * one word override this; wrappers that delegate to another {@code Stemmer} must forward this + * method so a multi-output delegate keeps its full result list. + * + * @param word The input word. Must not be {@code null}. Review Comment: What happens if null? Should be documented. ########## opennlp-api/src/main/java/opennlp/tools/stemmer/StemmerFactory.java: ########## @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.stemmer; + +/** + * An immutable, thread-safe factory for {@link Stemmer} instances. Review Comment: This is an interface and not an instance. It should not make claims about possible sub classes. ########## opennlp-api/src/main/java/opennlp/tools/stemmer/Stemmer.java: ########## @@ -17,10 +17,37 @@ package opennlp.tools.stemmer; +import java.util.List; + /** - * The stemmer is reducing a word to its stem. + * Reduces a word to its root form. + * + * <p>Thread safety is implementation-specific: check the implementation's documentation before + * sharing an instance across threads. Stateful engines mutate internal buffers on each + * {@link #stem(CharSequence)} call; when an implementation is not documented as thread-safe, + * share a {@link StemmerFactory} across threads and confine each {@code Stemmer} to one thread, + * or wrap the factory in a thread-local adapter when a single {@code Stemmer} reference must be + * shared.</p> */ public interface Stemmer { + /** + * Stems {@code word}. + * + * @param word The input word. Must not be {@code null}. Review Comment: What happens if it is null? Should be documented. ########## opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/CachingStemmer.java: ########## @@ -0,0 +1,136 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.stemmer; + +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import opennlp.tools.commons.ThreadSafe; +import opennlp.tools.util.OwnerOrPerThreadState; + +/** + * A {@link Stemmer} that memoizes word-to-stem mappings in a bounded per-thread LRU cache. + * + * <p>Natural language is Zipf-distributed: a small vocabulary accounts for most tokens, so the + * same words are stemmed over and over. Wrapping a stemmer in a {@code CachingStemmer} turns + * those repeats into a hash lookup instead of a full stemming pass.</p> + * + * <p>Each thread gets its own delegate stemmer (minted from the supplied {@link StemmerFactory}) Review Comment: Same comment as for the other instance regarding long running envs. ########## opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/snowball/SnowballStemmer.java: ########## @@ -17,68 +17,85 @@ package opennlp.tools.stemmer.snowball; +import java.util.function.Supplier; + +import opennlp.tools.commons.ThreadSafe; import opennlp.tools.stemmer.Stemmer; +import opennlp.tools.util.OwnerOrPerThreadState; +/** + * A {@link Stemmer} backed by the generated Snowball engines. Review Comment: This requires the same disclaimers as for the *ME classes regarding long running threads in app containers. ########## opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/CachingStemmer.java: ########## @@ -0,0 +1,136 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.stemmer; + +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import opennlp.tools.commons.ThreadSafe; +import opennlp.tools.util.OwnerOrPerThreadState; + +/** + * A {@link Stemmer} that memoizes word-to-stem mappings in a bounded per-thread LRU cache. + * + * <p>Natural language is Zipf-distributed: a small vocabulary accounts for most tokens, so the + * same words are stemmed over and over. Wrapping a stemmer in a {@code CachingStemmer} turns + * those repeats into a hash lookup instead of a full stemming pass.</p> + * + * <p>Each thread gets its own delegate stemmer (minted from the supplied {@link StemmerFactory}) + * and its own cache, following the {@link OwnerOrPerThreadState} pattern of the thread-safe + * {@code *ME} components. There is no cross-thread sharing at all, so instances are safe to share + * regardless of whether the factory's stemmers are. Memory is bounded by + * {@code capacity * averageWordLength} characters per thread that stems.</p> + * + * <p>Because the cache is keyed to the thread, the memoization pays off on threads that are + * reused across many words, such as a fixed platform-thread pool. On a virtual-thread-per-task + * executor every task starts with an empty cache, so repeats are only served within one task. + * Call {@link #clearThreadLocalState()} before returning a pooled thread that should not retain + * its delegate and cache.</p> + * + * <p>{@link #stemAll(CharSequence)} is forwarded to the delegate uncached, so multi-output + * engines keep their full result list.</p> + */ +@ThreadSafe +public final class CachingStemmer implements Stemmer { + + /** + * Covers the high-frequency vocabulary of most corpora while keeping the per-thread footprint + * small. + */ + public static final int DEFAULT_CAPACITY = 1024; + + private final OwnerOrPerThreadState<ThreadState> state; + + /** + * Creates a caching stemmer with the {@linkplain #DEFAULT_CAPACITY default capacity}. + * + * @param factory The factory that mints one delegate per thread. Must not be {@code null}. + * @throws IllegalArgumentException if {@code factory} is {@code null}. + */ + public CachingStemmer(StemmerFactory factory) { + this(factory, DEFAULT_CAPACITY); + } + + /** + * Creates a caching stemmer. + * + * @param factory The factory that mints one delegate per thread. Must not be {@code null}. + * @param capacity The maximum number of word-to-stem entries kept per thread; must be positive. + * @throws IllegalArgumentException if {@code factory} is {@code null} or {@code capacity} is + * not positive. + */ + public CachingStemmer(StemmerFactory factory, int capacity) { + if (factory == null) { + throw new IllegalArgumentException("factory must not be null"); + } + if (capacity <= 0) { + throw new IllegalArgumentException("capacity must be positive, got " + capacity); + } + this.state = new OwnerOrPerThreadState<>( + () -> new ThreadState(factory.newStemmer(), capacity), + threadState -> threadState.cache.clear()); + } + + @Override + public CharSequence stem(CharSequence word) { + final ThreadState ts = state.get(); + final String key = word.toString(); + final String cached = ts.cache.get(key); + if (cached != null) { + return cached; + } + // toString() detaches the result from any buffer the delegate may reuse internally. + final String stemmed = ts.delegate.stem(key).toString(); + ts.cache.put(key, stemmed); + return stemmed; + } + + @Override + public List<CharSequence> stemAll(CharSequence word) { + return state.get().delegate.stemAll(word); + } + + /** + * Removes this thread's delegate and cache to prevent classloader leaks in container + * environments. Call when the thread is returned to a pool or the stemmer is no longer needed, + * mirroring {@code clearThreadLocalState()} on the thread-safe {@code *ME} components. + */ + public void clearThreadLocalState() { + state.clearForCurrentThread(); + } + + private static final class ThreadState { + + private final Stemmer delegate; + private final LinkedHashMap<String, String> cache; Review Comment: Might also be a thing to rely on for caching here: https://github.com/apache/opennlp/blob/main/opennlp-api/src/main/java/opennlp/tools/util/jvm/StringInterner.java ########## opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/CachingStemmer.java: ########## @@ -0,0 +1,136 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.stemmer; + +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import opennlp.tools.commons.ThreadSafe; +import opennlp.tools.util.OwnerOrPerThreadState; + +/** + * A {@link Stemmer} that memoizes word-to-stem mappings in a bounded per-thread LRU cache. + * + * <p>Natural language is Zipf-distributed: a small vocabulary accounts for most tokens, so the + * same words are stemmed over and over. Wrapping a stemmer in a {@code CachingStemmer} turns + * those repeats into a hash lookup instead of a full stemming pass.</p> + * + * <p>Each thread gets its own delegate stemmer (minted from the supplied {@link StemmerFactory}) + * and its own cache, following the {@link OwnerOrPerThreadState} pattern of the thread-safe + * {@code *ME} components. There is no cross-thread sharing at all, so instances are safe to share + * regardless of whether the factory's stemmers are. Memory is bounded by + * {@code capacity * averageWordLength} characters per thread that stems.</p> + * + * <p>Because the cache is keyed to the thread, the memoization pays off on threads that are + * reused across many words, such as a fixed platform-thread pool. On a virtual-thread-per-task + * executor every task starts with an empty cache, so repeats are only served within one task. + * Call {@link #clearThreadLocalState()} before returning a pooled thread that should not retain + * its delegate and cache.</p> + * + * <p>{@link #stemAll(CharSequence)} is forwarded to the delegate uncached, so multi-output + * engines keep their full result list.</p> + */ +@ThreadSafe +public final class CachingStemmer implements Stemmer { + + /** + * Covers the high-frequency vocabulary of most corpora while keeping the per-thread footprint + * small. + */ + public static final int DEFAULT_CAPACITY = 1024; + + private final OwnerOrPerThreadState<ThreadState> state; + + /** + * Creates a caching stemmer with the {@linkplain #DEFAULT_CAPACITY default capacity}. + * + * @param factory The factory that mints one delegate per thread. Must not be {@code null}. + * @throws IllegalArgumentException if {@code factory} is {@code null}. + */ + public CachingStemmer(StemmerFactory factory) { + this(factory, DEFAULT_CAPACITY); + } + + /** + * Creates a caching stemmer. + * + * @param factory The factory that mints one delegate per thread. Must not be {@code null}. + * @param capacity The maximum number of word-to-stem entries kept per thread; must be positive. + * @throws IllegalArgumentException if {@code factory} is {@code null} or {@code capacity} is + * not positive. + */ + public CachingStemmer(StemmerFactory factory, int capacity) { + if (factory == null) { + throw new IllegalArgumentException("factory must not be null"); + } + if (capacity <= 0) { + throw new IllegalArgumentException("capacity must be positive, got " + capacity); + } + this.state = new OwnerOrPerThreadState<>( + () -> new ThreadState(factory.newStemmer(), capacity), + threadState -> threadState.cache.clear()); + } + + @Override + public CharSequence stem(CharSequence word) { + final ThreadState ts = state.get(); + final String key = word.toString(); + final String cached = ts.cache.get(key); + if (cached != null) { + return cached; + } + // toString() detaches the result from any buffer the delegate may reuse internally. + final String stemmed = ts.delegate.stem(key).toString(); + ts.cache.put(key, stemmed); + return stemmed; + } + + @Override + public List<CharSequence> stemAll(CharSequence word) { + return state.get().delegate.stemAll(word); + } + + /** + * Removes this thread's delegate and cache to prevent classloader leaks in container + * environments. Call when the thread is returned to a pool or the stemmer is no longer needed, + * mirroring {@code clearThreadLocalState()} on the thread-safe {@code *ME} components. + */ + public void clearThreadLocalState() { + state.clearForCurrentThread(); + } + + private static final class ThreadState { + + private final Stemmer delegate; + private final LinkedHashMap<String, String> cache; Review Comment: Can we have the possibiliy to force clean / empty the cache? -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
