This is an automated email from the ASF dual-hosted git repository.
mawiesne pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/opennlp.git
The following commit(s) were added to refs/heads/main by this push:
new 5a7b3aa6d OPENNLP-1863: Layered Term model — Term, TermAnalyzer (#1111)
5a7b3aa6d is described below
commit 5a7b3aa6d68a02c03f6a5fc2246f0829ef77847e
Author: Kristian Rickert <[email protected]>
AuthorDate: Mon Jul 6 08:29:19 2026 -0400
OPENNLP-1863: Layered Term model — Term, TermAnalyzer (#1111)
* OPENNLP-1863: Layered Term model: Term, TermAnalyzer (2b)
The token analysis layer split out of the former tokenizer PR (#1104) on
review request. A Term is
one token projected through the ordered Dimension stack (original, NFC,
NFKC, whitespace, dash, case
fold, accent fold, confusable fold, stem, lemma), keeping its source Span
and every intermediate
form; TermAnalyzer segments with the UAX #29 WordTokenizer (from 2a) and
applies the configured
dimension prefix. Restores Dimension's {@link Term}/{@link TermAnalyzer}
javadoc now that they exist.
Builds on the tokenizer in 2a.
* For broader context, see epic:
https://issues.apache.org/jira/browse/OPENNLP-1850
---
.../opennlp/tools/util/normalizer/Dimension.java | 10 +-
.../java/opennlp/tools/util/normalizer/Term.java | 148 +++++++
.../tools/util/normalizer/TermAnalyzer.java | 448 +++++++++++++++++++++
.../tools/util/normalizer/ConfusablesTest.java | 81 ++++
.../normalizer/TermAnalyzerMultilingualTest.java | 78 ++++
.../tools/util/normalizer/TermAnalyzerTest.java | 397 ++++++++++++++++++
.../opennlp/tools/util/normalizer/TermTest.java | 109 +++++
7 files changed, 1266 insertions(+), 5 deletions(-)
diff --git
a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/Dimension.java
b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/Dimension.java
index 6ad068471..a84fa0ff1 100644
---
a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/Dimension.java
+++
b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/Dimension.java
@@ -19,13 +19,13 @@ package opennlp.tools.util.normalizer;
import java.util.function.Supplier;
/**
- * A layer of the {@code Term} normalization stack, in increasing order of
aggressiveness. A
- * {@code TermAnalyzer} applies a configured prefix of these to each token;
the declaration order is
- * the canonical pipeline order, because the transforms do not commute (case
folding then accent
+ * A layer of the {@link Term} normalization stack, in increasing order of
aggressiveness. A
+ * {@link TermAnalyzer} applies a configured prefix of these to each token;
the declaration order is
+ * the pipeline order, because the transforms do not commute (case folding
then accent
* folding differs from the reverse for Turkish dotted/dotless i and the
German eszett).
*
* <p>This enum is the single definition of the character-level steps: each
one carries its default
- * {@link CharSequenceNormalizer}, which both {@code TermAnalyzer} and {@link
TextNormalizer} read
+ * {@link CharSequenceNormalizer}, which both {@link TermAnalyzer} and {@link
TextNormalizer} read
* from rather than re-listing. The default is resolved lazily, so loading
this enum does not eagerly
* initialize heavy data such as the confusables table.</p>
*
@@ -36,7 +36,7 @@ import java.util.function.Supplier;
*/
public enum Dimension {
- /** The original token text, the canonical source of truth. */
+ /** The original token text, the source of truth. */
ORIGINAL(null),
/** Unicode canonical composition (NFC); lossless under canonical
equivalence. */
diff --git
a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/Term.java
b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/Term.java
new file mode 100644
index 000000000..efe561d9f
--- /dev/null
+++
b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/Term.java
@@ -0,0 +1,148 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.tools.util.normalizer;
+
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+import opennlp.tools.util.Span;
+
+/**
+ * One token as a stack of normalization layers. The {@link #original()} form
is the source of
+ * truth; the other layers are derived, increasingly aggressive {@link
Dimension}s tuned for
+ * matching. The dimensions configured on the producing {@link TermAnalyzer}
are
+ * computed eagerly and cached; any other dimension is computed on first
request, applied on top of
+ * the {@link #normalized() configured form}, and then cached.
+ *
+ * <p>Because the original is always retained, aggressive folding is safe: a
match on a derived layer
+ * can always be reported in original coordinates through {@link #span()}.
Querying a configured
+ * layer, or {@link #peel() peeling} the last-applied one, is O(1); adding an
unconfigured dimension
+ * costs one transform on first touch and is O(1) thereafter.</p>
+ *
+ * <p>Instances are created by {@link TermAnalyzer} and are thread-safe as
long as the analyzer's
+ * configured transforms are (see {@link TermAnalyzer} for the stemmer
caveat). Concurrent first
+ * requests for the same unconfigured dimension may run its transform more
than once, but every
+ * thread observes the same cached value.</p>
+ */
+public final class Term {
+
+ private final TermAnalyzer analyzer;
+ private final Span span;
+ private final String posTag;
+ private final Map<Dimension, String> layers = new ConcurrentHashMap<>();
+
+ /**
+ * Creates a term and eagerly computes the analyzer's configured dimensions.
+ *
+ * @param analyzer The producing analyzer. Must not be {@code null}.
+ * @param original The original token text. Must not be {@code null}.
+ * @param span The source span of the token, or {@code null} for
pre-tokenized input.
+ * @param posTag The part-of-speech tag, or {@code null} when none is
available.
+ * @throws IllegalArgumentException if {@code analyzer} or {@code original}
is {@code null}.
+ * @throws IllegalStateException if a configured dimension needs an engine
or tag that is
+ * missing (see {@link TermAnalyzer#apply(Dimension, String, String)}).
+ */
+ Term(TermAnalyzer analyzer, String original, Span span, String posTag) {
+ if (analyzer == null) {
+ throw new IllegalArgumentException("analyzer must not be null");
+ }
+ if (original == null) {
+ throw new IllegalArgumentException("original must not be null");
+ }
+ this.analyzer = analyzer;
+ this.span = span;
+ this.posTag = posTag;
+ String value = original;
+ layers.put(Dimension.ORIGINAL, value);
+ for (final Dimension dimension : analyzer.dimensions()) {
+ value = analyzer.apply(dimension, value, posTag);
+ layers.put(dimension, value);
+ }
+ }
+
+ /**
+ * {@return the source span of this token, or {@code null} if it was
supplied as a pre-tokenized
+ * string} The span indexes into the text passed to {@link
TermAnalyzer#analyze(CharSequence)}.
+ */
+ public Span span() {
+ return span;
+ }
+
+ /**
+ * {@return the original token text}
+ */
+ public String original() {
+ return layers.get(Dimension.ORIGINAL);
+ }
+
+ /**
+ * {@return the token at the analyzer's final configured dimension; equal to
{@link #original()}
+ * when no dimensions were configured}
+ */
+ public String normalized() {
+ return at(analyzer.finalDimension());
+ }
+
+ /**
+ * Returns the token at {@code dimension}. Configured dimensions are cached;
an unconfigured
+ * dimension is computed by applying its transform to {@link #normalized()}
and then cached.
+ *
+ * <p>Note: an unconfigured dimension is applied on top of {@link
#normalized()} (the most
+ * aggressive configured layer), not spliced into pipeline order. Because
the transforms do not
+ * commute (see {@link Dimension}), requesting a dimension that ranks
<em>earlier</em> than the
+ * configured ones can differ from having configured it. For example, asking
for
+ * {@link Dimension#CASE_FOLD} on an analyzer configured only through {@link
Dimension#ACCENT_FOLD}
+ * case-folds the already accent-folded text, which is not the same as
case-folding first.
+ * Configure the dimension on the analyzer when pipeline order matters.</p>
+ *
+ * @param dimension The dimension to project to. Must not be {@code null}.
+ * @return The token at that dimension.
+ * @throws IllegalArgumentException if {@code dimension} is {@code null}.
+ * @throws IllegalStateException if the dimension needs an engine or tag
that was not configured
+ * (see {@link Dimension#STEM} and {@link Dimension#LEMMA}).
+ */
+ public String at(Dimension dimension) {
+ if (dimension == null) {
+ throw new IllegalArgumentException("dimension must not be null");
+ }
+ final String cached = layers.get(dimension);
+ if (cached != null) {
+ return cached;
+ }
+ // Computed outside computeIfAbsent so no map lock is held while the
transform runs: the
+ // transform routes through normalized(), which reads this map, and
ConcurrentHashMap forbids
+ // re-entrant use from a mapping function. Racing threads may both compute
the (deterministic)
+ // value; putIfAbsent keeps one winner for everyone.
+ final String value = analyzer.apply(dimension, normalized(), posTag);
+ final String winner = layers.putIfAbsent(dimension, value);
+ return winner != null ? winner : value;
+ }
+
+ /**
+ * {@return the token at the dimension just below the final configured one}
This is the
+ * last-applied layer removed (for example the form before stemming when
{@link Dimension#STEM}
+ * is the final dimension); equal to {@link #original()} when at most one
dimension is configured.
+ */
+ public String peel() {
+ final List<Dimension> dimensions = analyzer.dimensions();
+ if (dimensions.size() < 2) {
+ return original();
+ }
+ return at(dimensions.get(dimensions.size() - 2));
+ }
+}
diff --git
a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/TermAnalyzer.java
b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/TermAnalyzer.java
new file mode 100644
index 000000000..39f213aaa
--- /dev/null
+++
b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/TermAnalyzer.java
@@ -0,0 +1,448 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.tools.util.normalizer;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.EnumMap;
+import java.util.EnumSet;
+import java.util.List;
+import java.util.Locale;
+import java.util.Set;
+
+import opennlp.tools.lemmatizer.Lemmatizer;
+import opennlp.tools.stemmer.Stemmer;
+import opennlp.tools.tokenize.uax29.WordTokenizer;
+import opennlp.tools.util.Span;
+
+/**
+ * Builds {@link Term}s by segmenting text and applying a configured stack of
normalization
+ * {@link Dimension}s to each token. The analyzer is the configuration; each
{@link Term} is the
+ * layered result for one token, with the configured dimensions computed
eagerly and any other
+ * dimension computed lazily on first request.
+ *
+ * <p>Segmentation uses the Unicode {@linkplain WordTokenizer UAX #29
word tokenizer}, so the
+ * input does not need to be pre-tokenized. The character-level dimensions
({@link Dimension#NFC}
+ * through {@link Dimension#CONFUSABLE_FOLD}) have built-in defaults; {@link
Dimension#STEM} and
+ * {@link Dimension#LEMMA} are enabled by supplying a {@link Stemmer} or
{@link Lemmatizer}.</p>
+ *
+ * <p>An instance is immutable and is thread-safe when its configured
transforms are. The built-in
+ * character normalizers are stateless, but the Snowball stemmers are not, so
an analyzer configured
+ * with a {@link Stemmer} (for example through {@code
NormalizationProfile.matchingAnalyzer()}) should
+ * not be shared across threads when {@link Dimension#STEM} is used. Build one
with
+ * {@link #builder()}.</p>
+ */
+public final class TermAnalyzer {
+
+ private final List<Dimension> chain;
+ private final Dimension finalDimension;
+ private final EnumMap<Dimension, CharSequenceNormalizer> transforms;
+ private final Stemmer stemmer;
+ private final Lemmatizer lemmatizer;
+ private final WordTokenizer tokenizer;
+
+ private TermAnalyzer(Builder builder) {
+ final List<Dimension> ordered = new ArrayList<>(builder.chain);
+ Collections.sort(ordered); // pipeline order (enum declaration order)
+ this.chain = List.copyOf(ordered);
+ this.finalDimension = ordered.isEmpty() ? Dimension.ORIGINAL :
ordered.get(ordered.size() - 1);
+ // Only the per-analyzer overrides from the builder; the defaults live on
Dimension itself.
+ this.transforms = new EnumMap<>(builder.transforms);
+ this.stemmer = builder.stemmer;
+ this.lemmatizer = builder.lemmatizer;
+ this.tokenizer = builder.tokenizer;
+ }
+
+ /**
+ * {@return a new {@link Builder}} The builder starts with no dimensions
enabled and the default
+ * UAX #29 word tokenizer; enable dimensions and set a stemmer,
lemmatizer, or tokenizer on it,
+ * then call {@link Builder#build()}.
+ */
+ public static Builder builder() {
+ return new Builder();
+ }
+
+ /**
+ * Segments {@code text} with the UAX #29 word tokenizer and returns
one {@link Term} per
+ * word token, in order. The terms carry no part-of-speech tag, so {@link
Dimension#LEMMA} cannot be
+ * computed from this entry point: if a lemmatizer is configured, this
method throws -- use
+ * {@link #analyze(String[], String[])} when lemmas are needed.
+ *
+ * @param text The text to analyze. Must not be {@code null}.
+ * @return The terms.
+ * @throws IllegalArgumentException if {@code text} is {@code null}.
+ * @throws IllegalStateException if {@link Dimension#LEMMA} is configured,
because no
+ * part-of-speech tags are available from raw text.
+ */
+ public List<Term> analyze(CharSequence text) {
+ if (text == null) {
+ throw new IllegalArgumentException("text must not be null");
+ }
+ if (chain.contains(Dimension.LEMMA)) {
+ throw new IllegalStateException("Dimension LEMMA requires part-of-speech
tags, which"
+ + " analyze(CharSequence) cannot supply; use analyze(tokens, tags)");
+ }
+ final List<Span> spans = tokenizer.tokenizeSpans(text);
+ final List<Term> terms = new ArrayList<>(spans.size());
+ for (final Span span : spans) {
+ terms.add(new Term(this, span.getCoveredText(text).toString(), span,
null));
+ }
+ return terms;
+ }
+
+ /**
+ * Returns one {@link Term} per supplied token, attaching the matching
part-of-speech tag so that
+ * {@link Dimension#LEMMA} can be computed. The terms have no source span.
+ *
+ * @param tokens The tokens. Must not be {@code null} or contain {@code
null} elements.
+ * @param tags The part-of-speech tag for each token; must be the same
length as {@code tokens}
+ * and must not be {@code null}. A {@code null} tag is only
acceptable when
+ * {@link Dimension#LEMMA} is not computed for that token.
+ * @return The terms.
+ * @throws IllegalArgumentException if {@code tokens} or {@code tags} is
{@code null}, if they
+ * differ in length, or if {@code tokens} contains a {@code null}
element.
+ */
+ public List<Term> analyze(String[] tokens, String[] tags) {
+ if (tokens == null) {
+ throw new IllegalArgumentException("tokens must not be null");
+ }
+ if (tags == null) {
+ throw new IllegalArgumentException("tags must not be null");
+ }
+ if (tokens.length != tags.length) {
+ throw new IllegalArgumentException(
+ "tokens and tags must be the same length, got " + tokens.length + "
and " + tags.length);
+ }
+ final List<Term> terms = new ArrayList<>(tokens.length);
+ for (int i = 0; i < tokens.length; i++) {
+ if (tokens[i] == null) {
+ throw new IllegalArgumentException("tokens[" + i + "] is null");
+ }
+ terms.add(new Term(this, tokens[i], null, tags[i]));
+ }
+ return terms;
+ }
+
+ /**
+ * {@return the configured dimensions that are computed eagerly, in pipeline
order} The list
+ * never includes {@link Dimension#ORIGINAL}, which is always present.
+ */
+ public List<Dimension> dimensions() {
+ return chain;
+ }
+
+ /**
+ * {@return the last configured dimension in pipeline order, or {@link
Dimension#ORIGINAL} when
+ * none are configured} This is the layer {@link Term#normalized()} reports.
+ */
+ Dimension finalDimension() {
+ return finalDimension;
+ }
+
+ /**
+ * Applies one dimension's transform to a single token value.
+ *
+ * @param dimension The dimension whose transform to apply.
+ * @param input The token value to transform.
+ * @param posTag The token's part-of-speech tag; only read by {@link
Dimension#LEMMA} and may
+ * be {@code null} otherwise.
+ * @return The transformed value; never {@code null}.
+ * @throws IllegalStateException if a token-level dimension was requested
without the engine (or
+ * tag) it needs: {@link Dimension#STEM} without a {@link Stemmer},
{@link Dimension#LEMMA}
+ * without a {@link Lemmatizer} or without a tag, or a lemmatizer that
returns no lemma. Also
+ * thrown for a character-level dimension with neither a default nor a
configured normalizer.
+ */
+ String apply(Dimension dimension, String input, String posTag) {
+ switch (dimension) {
+ case ORIGINAL:
+ return input;
+ case STEM:
+ if (stemmer == null) {
+ throw new IllegalStateException(
+ "Dimension STEM requires a Stemmer; configure it with
builder().stem(...)");
+ }
+ return stemmer.stem(input).toString();
+ case LEMMA:
+ if (lemmatizer == null) {
+ throw new IllegalStateException(
+ "Dimension LEMMA requires a Lemmatizer; configure it with
builder().lemmatize(...)");
+ }
+ if (posTag == null) {
+ throw new IllegalStateException("Dimension LEMMA requires a
part-of-speech tag, but the"
+ + " tag for token '" + input + "' was null; use analyze(tokens,
tags) with a"
+ + " non-null tag per token");
+ }
+ final String[] lemmas = lemmatizer.lemmatize(new String[] {input}, new
String[] {posTag});
+ if (lemmas == null || lemmas.length == 0 || lemmas[0] == null) {
+ // A contract-violating Lemmatizer must fail loud here: a null
cached under LEMMA would
+ // read as "absent" in Term.at's lazy cache and recompute through
normalized() forever,
+ // surfacing as a StackOverflowError far from the cause.
+ throw new IllegalStateException(
+ "The Lemmatizer returned no lemma for token '" + input + "'");
+ }
+ return lemmas[0];
+ default:
+ // A builder override wins; otherwise the dimension's own default
normalizer.
+ final CharSequenceNormalizer normalizer =
transforms.containsKey(dimension)
+ ? transforms.get(dimension) : dimension.defaultNormalizer();
+ if (normalizer == null) {
+ throw new IllegalStateException("Dimension " + dimension + " has no
default normalizer; "
+ + "configure it with builder().transform(" + dimension + ",
...)");
+ }
+ return normalizer.normalize(input).toString();
+ }
+ }
+
+ /** A builder for {@link TermAnalyzer}. */
+ public static final class Builder {
+
+ private final EnumSet<Dimension> chain = EnumSet.noneOf(Dimension.class);
+ private final EnumMap<Dimension, CharSequenceNormalizer> transforms =
+ new EnumMap<>(Dimension.class);
+ private Stemmer stemmer;
+ private Lemmatizer lemmatizer;
+ private WordTokenizer tokenizer = new WordTokenizer();
+
+ private Builder() {
+ }
+
+ /**
+ * Enables {@link Dimension#NFC}.
+ *
+ * @return this builder
+ */
+ public Builder nfc() {
+ chain.add(Dimension.NFC);
+ return this;
+ }
+
+ /**
+ * Enables {@link Dimension#NFKC}.
+ *
+ * @return this builder
+ */
+ public Builder nfkc() {
+ chain.add(Dimension.NFKC);
+ return this;
+ }
+
+ /**
+ * Enables {@link Dimension#WHITESPACE}.
+ *
+ * @return this builder
+ */
+ public Builder whitespace() {
+ chain.add(Dimension.WHITESPACE);
+ return this;
+ }
+
+ /**
+ * Enables {@link Dimension#WHITESPACE} with a specific normalizer,
choosing the fold target and
+ * behavior. For a custom class and target use a {@link CharClass} method
reference, for example
+ * {@code whitespace(CharClass.of(members, replacement)::collapse)}.
+ *
+ * @param normalizer The whitespace normalizer to use. Must not be {@code
null}.
+ * @return this builder
+ * @throws IllegalArgumentException if {@code normalizer} is {@code null}.
+ */
+ public Builder whitespace(CharSequenceNormalizer normalizer) {
+ return transform(Dimension.WHITESPACE, normalizer);
+ }
+
+ /**
+ * Enables {@link Dimension#DASH}.
+ *
+ * @return this builder
+ */
+ public Builder dash() {
+ chain.add(Dimension.DASH);
+ return this;
+ }
+
+ /**
+ * Enables {@link Dimension#DASH} with a specific normalizer (a custom
dash set or target).
+ *
+ * @param normalizer The dash normalizer to use. Must not be {@code null}.
+ * @return this builder
+ * @throws IllegalArgumentException if {@code normalizer} is {@code null}.
+ */
+ public Builder dash(CharSequenceNormalizer normalizer) {
+ return transform(Dimension.DASH, normalizer);
+ }
+
+ /**
+ * Enables {@link Dimension#CASE_FOLD}.
+ *
+ * @return this builder
+ */
+ public Builder caseFold() {
+ chain.add(Dimension.CASE_FOLD);
+ return this;
+ }
+
+ /**
+ * Enables {@link Dimension#CASE_FOLD} using the given locale's case rules
(for example Turkish
+ * dotted/dotless i), instead of the default {@link Locale#ROOT}.
+ *
+ * @param locale The locale whose case rules to apply. Must not be {@code
null}.
+ * @return this builder
+ * @throws IllegalArgumentException if {@code locale} is {@code null}.
+ */
+ public Builder caseFold(Locale locale) {
+ if (locale == null) {
+ throw new IllegalArgumentException("locale must not be null");
+ }
+ return transform(Dimension.CASE_FOLD,
CaseFoldCharSequenceNormalizer.getInstance(locale));
+ }
+
+ /**
+ * Enables {@link Dimension#ACCENT_FOLD}.
+ *
+ * @return this builder
+ */
+ public Builder accentFold() {
+ chain.add(Dimension.ACCENT_FOLD);
+ return this;
+ }
+
+ /**
+ * Enables {@link Dimension#ACCENT_FOLD} restricted to a specific set of
scripts, instead of the
+ * default Latin/Greek/Cyrillic.
+ *
+ * @param foldScripts The scripts whose diacritics to fold. Must not
be {@code null} or
+ * contain {@code null} elements.
+ * @param foldStrokeLetters Whether to also fold stroke letters such as
o-slash and l-stroke.
+ * @return this builder
+ * @throws IllegalArgumentException if {@code foldScripts} is {@code null}
or contains a
+ * {@code null} element.
+ */
+ public Builder accentFold(Set<Character.UnicodeScript> foldScripts,
boolean foldStrokeLetters) {
+ if (foldScripts == null) {
+ throw new IllegalArgumentException("foldScripts must not be null");
+ }
+ return transform(Dimension.ACCENT_FOLD,
+ new AccentFoldCharSequenceNormalizer(foldScripts,
foldStrokeLetters));
+ }
+
+ /**
+ * Enables {@link Dimension#CONFUSABLE_FOLD}.
+ *
+ * @return this builder
+ */
+ public Builder confusableFold() {
+ chain.add(Dimension.CONFUSABLE_FOLD);
+ return this;
+ }
+
+ /**
+ * Enables a character-level dimension with a specific normalizer,
overriding its default (for
+ * example a locale-specific case fold for a language profile).
+ *
+ * @param dimension The character-level dimension to enable. Must not be
{@code null}.
+ * @param normalizer The normalizer to use for it. Must not be {@code
null}.
+ * @return this builder
+ * @throws IllegalArgumentException if {@code dimension} or {@code
normalizer} is {@code null},
+ * or if {@code dimension} is {@link Dimension#ORIGINAL}, {@link
Dimension#STEM}, or
+ * {@link Dimension#LEMMA}.
+ */
+ public Builder transform(Dimension dimension, CharSequenceNormalizer
normalizer) {
+ if (dimension == null) {
+ throw new IllegalArgumentException("dimension must not be null");
+ }
+ if (normalizer == null) {
+ throw new IllegalArgumentException("normalizer must not be null");
+ }
+ if (dimension == Dimension.ORIGINAL || dimension == Dimension.STEM
+ || dimension == Dimension.LEMMA) {
+ throw new IllegalArgumentException(
+ "transform(...) only applies to character-level dimensions, not "
+ dimension);
+ }
+ transforms.put(dimension, normalizer);
+ chain.add(dimension);
+ return this;
+ }
+
+ /**
+ * Enables {@link Dimension#STEM} through the given stemmer.
+ *
+ * @param value The stemmer. Must not be {@code null}.
+ * @return this builder
+ * @throws IllegalArgumentException if {@code value} is {@code null}.
+ */
+ public Builder stem(Stemmer value) {
+ if (value == null) {
+ throw new IllegalArgumentException("stemmer must not be null");
+ }
+ this.stemmer = value;
+ chain.add(Dimension.STEM);
+ return this;
+ }
+
+ /**
+ * Enables {@link Dimension#LEMMA} through the given lemmatizer.
+ *
+ * @param value The lemmatizer. Must not be {@code null}.
+ * @return this builder
+ * @throws IllegalArgumentException if {@code value} is {@code null}.
+ */
+ public Builder lemmatize(Lemmatizer value) {
+ if (value == null) {
+ throw new IllegalArgumentException("lemmatizer must not be null");
+ }
+ this.lemmatizer = value;
+ chain.add(Dimension.LEMMA);
+ return this;
+ }
+
+ /**
+ * Sets the tokenizer used by {@link TermAnalyzer#analyze(CharSequence)}.
+ *
+ * @param value The tokenizer. Must not be {@code null}.
+ * @return this builder
+ * @throws IllegalArgumentException if {@code value} is {@code null}.
+ */
+ public Builder tokenizer(WordTokenizer value) {
+ if (value == null) {
+ throw new IllegalArgumentException("tokenizer must not be null");
+ }
+ this.tokenizer = value;
+ return this;
+ }
+
+ /**
+ * Sets the maximum token length of the tokenizer used by
+ * {@link TermAnalyzer#analyze(CharSequence)}. Convenience for
+ * {@code tokenizer(new WordTokenizer(maxTokenLength))}.
+ *
+ * @param maxTokenLength The maximum number of characters in a token. Must
be at least
+ * {@code 1}.
+ * @return this builder
+ * @throws IllegalArgumentException if {@code maxTokenLength} is less than
{@code 1}.
+ */
+ public Builder maxTokenLength(int maxTokenLength) {
+ this.tokenizer = new WordTokenizer(maxTokenLength);
+ return this;
+ }
+
+ /**
+ * {@return a new {@link TermAnalyzer} with this configuration}
+ */
+ public TermAnalyzer build() {
+ return new TermAnalyzer(this);
+ }
+ }
+}
diff --git
a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/ConfusablesTest.java
b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/ConfusablesTest.java
new file mode 100644
index 000000000..262fe5aa9
--- /dev/null
+++
b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/ConfusablesTest.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.tools.util.normalizer;
+
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+public class ConfusablesTest {
+
+ private static String cp(int codePoint) {
+ return new String(Character.toChars(codePoint));
+ }
+
+ @Test
+ void testCyrillicLetterIsConfusableWithLatin() {
+ final String cyrillicA = cp(0x0430); // CYRILLIC SMALL LETTER A, looks
like Latin 'a'
+ assertTrue(Confusables.confusable(cyrillicA, "a"));
+ assertFalse(Confusables.confusable(cyrillicA, "b"));
+ }
+
+ @Test
+ void testHomoglyphSpoofWordReducesToLatinSpelling() {
+ final String spoof = "p" + cp(0x0430) + "yp" + cp(0x0430) + "l"; // paypal
with Cyrillic a's
+ assertTrue(Confusables.confusable(spoof, "paypal"));
+ assertEquals(Confusables.skeleton("paypal"), Confusables.skeleton(spoof));
+ }
+
+ @Test
+ void testHorizontalEllipsisFoldsToThreeFullStops() {
+ assertEquals(Confusables.skeleton("..."),
Confusables.skeleton(cp(0x2026)));
+ assertTrue(Confusables.confusable(cp(0x2026), "..."));
+ }
+
+ @Test
+ void testDistinctWordsAreNotConfusable() {
+ assertFalse(Confusables.confusable("cat", "dog"));
+ }
+
+ @Test
+ void testSkeletonIsIdempotent() {
+ final String skeleton = Confusables.skeleton(cp(0x0430) + "bc");
+ assertEquals(skeleton, Confusables.skeleton(skeleton));
+ }
+
+ @Test
+ void testNormalizerProducesTheSkeleton() {
+ final String spoof = "p" + cp(0x0430) + "yp" + cp(0x0430) + "l";
+ assertEquals(Confusables.skeleton(spoof),
+
ConfusableSkeletonCharSequenceNormalizer.getInstance().normalize(spoof).toString());
+ }
+
+ @Test
+ void testMultipleCyrillicLookalikesFold() {
+ final String spoof = "d" + cp(0x0430) + "t" + cp(0x0430); // "data" with
Cyrillic a's
+ assertEquals(Confusables.skeleton("data"), Confusables.skeleton(spoof));
+ }
+
+ @Test
+ void testTermConfusableFoldDimension() {
+ final String spoof = "p" + cp(0x0430) + "yp" + cp(0x0430) + "l";
+ final TermAnalyzer analyzer =
TermAnalyzer.builder().confusableFold().build();
+ assertEquals(Confusables.skeleton("paypal"),
analyzer.analyze(spoof).get(0).normalized());
+ }
+}
diff --git
a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/TermAnalyzerMultilingualTest.java
b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/TermAnalyzerMultilingualTest.java
new file mode 100644
index 000000000..cc567001d
--- /dev/null
+++
b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/TermAnalyzerMultilingualTest.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.tools.util.normalizer;
+
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+/**
+ * Normalization behavior of {@link TermAnalyzer} across a range of scripts
and diacritics: German
+ * umlauts (the language-specific digraph fold), Romance-language accents (the
generic accent fold),
+ * Cyrillic case folding, and CJK (Japanese) canonical and compatibility
forms. Source strings are
+ * built from code points to keep this file ASCII-only.
+ */
+public class TermAnalyzerMultilingualTest {
+
+ private static String cp(int... codePoints) {
+ return new String(codePoints, 0, codePoints.length);
+ }
+
+ private static String normalized(TermAnalyzer analyzer, String text) {
+ return analyzer.analyze(text).get(0).normalized();
+ }
+
+ @Test
+ void germanUmlautsFoldToTheirDigraphs() {
+ // The German-specific fold expands the umlauts and eszett (ue/oe/ae/ss),
where the generic
+ // accent fold would merely strip the diaeresis.
+ final TermAnalyzer analyzer = TermAnalyzer.builder().caseFold()
+ .transform(Dimension.ACCENT_FOLD,
GermanUmlautCharSequenceNormalizer.getInstance()).build();
+ assertEquals("gruesse", normalized(analyzer, "GR" + cp(0x00DC) +
cp(0x00DF) + "E")); // GRUesseE
+ assertEquals("ueber", normalized(analyzer, cp(0x00DC) + "ber"));
// Ueber
+ }
+
+ @Test
+ void genericAccentFoldStripsRomanceDiacritics() {
+ final TermAnalyzer analyzer =
TermAnalyzer.builder().caseFold().accentFold().build();
+ assertEquals("eleve", normalized(analyzer, cp(0x00C9) + "l" + cp(0x00E8) +
"ve")); // Eleve (fr)
+ assertEquals("nino", normalized(analyzer, "Ni" + cp(0x00F1) + "o"));
// Nino (es)
+ assertEquals("cancion", normalized(analyzer, "Canci" + cp(0x00F3) + "n"));
// Cancion
+ }
+
+ @Test
+ void cyrillicCaseFolds() {
+ final TermAnalyzer analyzer = TermAnalyzer.builder().caseFold().build();
+ // MOSKVA -> moskva
+ final String moscowUpper = cp(0x041C, 0x041E, 0x0421, 0x041A, 0x0412,
0x0410);
+ final String moscowLower = cp(0x043C, 0x043E, 0x0441, 0x043A, 0x0432,
0x0430);
+ assertEquals(moscowLower, normalized(analyzer, moscowUpper));
+ }
+
+ @Test
+ void japaneseFullWidthLetterFoldsUnderNfkc() {
+ // NFKC maps full-width compatibility letters to their canonical ASCII
form; case fold lowers it.
+ final TermAnalyzer analyzer =
TermAnalyzer.builder().nfkc().caseFold().build();
+ assertEquals("a", normalized(analyzer, cp(0xFF21))); // fullwidth A -> a
+ }
+
+ @Test
+ void japaneseKanjiPassesThroughNfc() {
+ final TermAnalyzer analyzer = TermAnalyzer.builder().nfc().build();
+ assertEquals(cp(0x6771), normalized(analyzer, cp(0x6771))); // kanji for
"east" unchanged
+ }
+}
diff --git
a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/TermAnalyzerTest.java
b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/TermAnalyzerTest.java
new file mode 100644
index 000000000..d7fd52474
--- /dev/null
+++
b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/TermAnalyzerTest.java
@@ -0,0 +1,397 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.tools.util.normalizer;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Locale;
+import java.util.Set;
+import java.util.concurrent.CyclicBarrier;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+
+import org.junit.jupiter.api.Test;
+
+import opennlp.tools.lemmatizer.Lemmatizer;
+import opennlp.tools.stemmer.PorterStemmer;
+import opennlp.tools.util.Span;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertSame;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+public class TermAnalyzerTest {
+
+ private static String cp(int codePoint) {
+ return new String(Character.toChars(codePoint));
+ }
+
+ @Test
+ void testNoDimensionsLeavesTokenUnchanged() {
+ final TermAnalyzer analyzer = TermAnalyzer.builder().build();
+ final Term term = analyzer.analyze("Hello").get(0);
+ assertEquals("Hello", term.original());
+ assertEquals("Hello", term.normalized());
+ assertEquals("Hello", term.peel());
+ assertEquals(List.of(), analyzer.dimensions());
+ }
+
+ @Test
+ void testChainAppliesInPipelineOrderRegardlessOfBuilderOrder() {
+ // accentFold added before caseFold, but the pipeline order is caseFold
then accentFold.
+ final TermAnalyzer analyzer =
TermAnalyzer.builder().accentFold().caseFold().build();
+ assertEquals(List.of(Dimension.CASE_FOLD, Dimension.ACCENT_FOLD),
analyzer.dimensions());
+ final String input = "CAF" + cp(0x00C9); // CAFE with capital acute E
+ final Term term = analyzer.analyze(input).get(0);
+ assertEquals(input, term.original());
+ assertEquals("cafe", term.normalized());
+ assertEquals("caf" + cp(0x00E9), term.peel()); // before accent folding:
lower-case, acute kept
+ }
+
+ @Test
+ void testStemIsTheTopLayer() {
+ final TermAnalyzer analyzer =
+ TermAnalyzer.builder().caseFold().stem(new PorterStemmer()).build();
+ final Term term = analyzer.analyze("Running").get(0);
+ assertEquals("running", term.peel()); // case-folded form, before stemming
+ assertEquals("run", term.normalized());
+ assertEquals("run", term.at(Dimension.STEM));
+ }
+
+ @Test
+ void testUnconfiguredCharDimensionComputedLazily() {
+ final TermAnalyzer analyzer = TermAnalyzer.builder().build();
+ final Term term = analyzer.analyze("HELLO").get(0);
+ assertEquals("HELLO", term.normalized());
+ assertEquals("hello", term.at(Dimension.CASE_FOLD)); // lazily added on
top of the final form
+ }
+
+ @Test
+ void testStemDimensionWithoutStemmerFailsLoudly() {
+ final TermAnalyzer analyzer = TermAnalyzer.builder().caseFold().build();
+ final Term term = analyzer.analyze("running").get(0);
+ assertThrows(IllegalStateException.class, () -> term.at(Dimension.STEM));
+ }
+
+ @Test
+ void testLemmaWithoutLemmatizerFailsLoudly() {
+ final TermAnalyzer analyzer = TermAnalyzer.builder().build();
+ final Term term = analyzer.analyze("running").get(0);
+ assertThrows(IllegalStateException.class, () -> term.at(Dimension.LEMMA));
+ }
+
+ @Test
+ void testAnalyzeTextProducesSpans() {
+ final TermAnalyzer analyzer = TermAnalyzer.builder().caseFold().build();
+ final List<Term> terms = analyzer.analyze("The Cats");
+ assertEquals(2, terms.size());
+ assertEquals("The", terms.get(0).original());
+ assertEquals("the", terms.get(0).normalized());
+ assertEquals(new Span(0, 3), terms.get(0).span());
+ assertEquals("Cats", terms.get(1).original());
+ assertEquals(new Span(4, 8), terms.get(1).span());
+ }
+
+ @Test
+ void testAnalyzeTokensHasNoSpan() {
+ final TermAnalyzer analyzer = TermAnalyzer.builder().caseFold().build();
+ final List<Term> terms = analyzer.analyze(new String[] {"Cats"}, new
String[] {"NNS"});
+ assertNull(terms.get(0).span());
+ assertEquals("cats", terms.get(0).normalized());
+ }
+
+ @Test
+ void testAnalyzeTokensRejectsLengthMismatch() {
+ final TermAnalyzer analyzer = TermAnalyzer.builder().build();
+ assertThrows(IllegalArgumentException.class,
+ () -> analyzer.analyze(new String[] {"a", "b"}, new String[] {"X"}));
+ }
+
+ @Test
+ void testTransformRejectsNonCharacterDimension() {
+ assertThrows(IllegalArgumentException.class, () -> TermAnalyzer.builder()
+ .transform(Dimension.STEM,
CaseFoldCharSequenceNormalizer.getInstance()));
+ }
+
+ @Test
+ void testLemmaWithLemmatizerAndTag() {
+ final Lemmatizer lemmatizer = new Lemmatizer() {
+ @Override
+ public String[] lemmatize(String[] tokens, String[] tags) {
+ return new String[] {"be"};
+ }
+
+ @Override
+ public List<List<String>> lemmatize(List<String> tokens, List<String>
tags) {
+ return List.of(List.of("be"));
+ }
+ };
+ final TermAnalyzer analyzer =
+ TermAnalyzer.builder().caseFold().lemmatize(lemmatizer).build();
+ final Term term = analyzer.analyze(new String[] {"was"}, new String[]
{"VBD"}).get(0);
+ assertEquals("be", term.normalized());
+ }
+
+ @Test
+ void testLemmatizerReturningNullFailsLoudlyInsteadOfOverflowing() {
+ // A contract-violating Lemmatizer that returns a null lemma must surface
as a clear
+ // IllegalStateException. Before this guard the null was cached under
LEMMA, read as "absent"
+ // by Term.at's lazy cache, and recomputed through normalized() forever,
surfacing as a
+ // StackOverflowError far from the cause.
+ final Lemmatizer broken = new Lemmatizer() {
+ @Override
+ public String[] lemmatize(String[] tokens, String[] tags) {
+ return new String[] {null};
+ }
+
+ @Override
+ public List<List<String>> lemmatize(List<String> tokens, List<String>
tags) {
+ return List.of();
+ }
+ };
+ final TermAnalyzer analyzer =
TermAnalyzer.builder().lemmatize(broken).build();
+ // Configured dimensions are computed eagerly in the Term constructor, so
the guard fires
+ // already during analyze, as close to the misbehaving Lemmatizer as
possible.
+ final IllegalStateException e = assertThrows(IllegalStateException.class,
+ () -> analyzer.analyze(new String[] {"was"}, new String[] {"VBD"}));
+ assertTrue(e.getMessage().contains("was"));
+ }
+
+ @Test
+ void testAnalyzeCharSequenceFailsLoudlyWhenLemmaConfigured() {
+ // analyze(CharSequence) has no POS tags, so a configured LEMMA layer
cannot be satisfied; it
+ // fails loud rather than silently dropping the layer. Callers needing
lemmas use analyze(tokens,
+ // tags).
+ final Lemmatizer lemmatizer = new Lemmatizer() {
+ @Override
+ public String[] lemmatize(String[] tokens, String[] tags) {
+ return tokens.clone();
+ }
+
+ @Override
+ public List<List<String>> lemmatize(List<String> tokens, List<String>
tags) {
+ return List.of(tokens);
+ }
+ };
+ final TermAnalyzer analyzer =
TermAnalyzer.builder().lemmatize(lemmatizer).build();
+ final IllegalStateException e = assertThrows(IllegalStateException.class,
+ () -> analyzer.analyze("running"));
+ assertTrue(e.getMessage().contains("part-of-speech"), e.getMessage());
+ }
+
+ @Test
+ void testConfusableFoldComposesWithCaseFold() {
+ final TermAnalyzer analyzer =
TermAnalyzer.builder().caseFold().confusableFold().build();
+ final String spoof = "P" + cp(0x0430) + "yp" + cp(0x0430) + "l"; // Paypal
with Cyrillic a's
+ assertEquals(Confusables.skeleton("paypal"),
analyzer.analyze(spoof).get(0).normalized());
+ }
+
+ @Test
+ void testAtIsMemoized() {
+ final TermAnalyzer analyzer = TermAnalyzer.builder().build();
+ final Term term = analyzer.analyze("HELLO").get(0);
+ final String first = term.at(Dimension.CASE_FOLD);
+ assertSame(first, term.at(Dimension.CASE_FOLD));
+ }
+
+ @Test
+ void testWhitespaceTargetIsConfigurable() {
+ final CharClass lineFold = CharClass.of(CodePointSet.of('\n', '\t'), '\n');
+ final TermAnalyzer analyzer =
TermAnalyzer.builder().whitespace(lineFold::collapse).build();
+ final Term term = analyzer.analyze(new String[] {"a\n\n\tb"}, new String[]
{"X"}).get(0);
+ assertEquals("a\nb", term.normalized());
+ }
+
+ @Test
+ void testCaseFoldLocaleAppliesTurkishRules() {
+ final TermAnalyzer analyzer =
+ TermAnalyzer.builder().caseFold(Locale.forLanguageTag("tr")).build();
+ assertEquals(cp(0x0131), analyzer.analyze("I").get(0).normalized()); //
dotless lowercase i
+ }
+
+ @Test
+ void testAccentFoldScopeFoldsLatin() {
+ final TermAnalyzer analyzer = TermAnalyzer.builder()
+ .accentFold(Set.of(Character.UnicodeScript.LATIN), false).build();
+ assertEquals("cafe", analyzer.analyze("caf" +
cp(0x00E9)).get(0).normalized()); // cafe + acute
+ }
+
+ @Test
+ void testMaxTokenLengthChopsTokens() {
+ final List<Term> terms =
TermAnalyzer.builder().maxTokenLength(3).build().analyze("abcdefg");
+ assertEquals(3, terms.size());
+ assertEquals("abc", terms.get(0).original());
+ assertEquals("def", terms.get(1).original());
+ assertEquals("g", terms.get(2).original());
+ }
+
+ @Test
+ void testAnalyzeEmptyTextProducesNoTerms() {
+ assertEquals(List.of(),
TermAnalyzer.builder().caseFold().build().analyze(""));
+ }
+
+ @Test
+ void testWhitespaceOnlyInputHasNoWordTerms() {
+ assertEquals(List.of(), TermAnalyzer.builder().build().analyze(" \t "));
+ }
+
+ @Test
+ void testAtDimensionBelowFinalIsAppliedOnTop() {
+ // Final dimension is STEM; asking for NFC applies it on top of the stem
(documented behavior).
+ final TermAnalyzer analyzer =
+ TermAnalyzer.builder().caseFold().stem(new PorterStemmer()).build();
+ final Term term = analyzer.analyze("Running").get(0);
+ assertEquals("run", term.normalized());
+ assertEquals("run", term.at(Dimension.NFC));
+ }
+
+ @Test
+ void testAnalyzeTextRejectsNull() {
+ final TermAnalyzer analyzer = TermAnalyzer.builder().build();
+ assertThrows(IllegalArgumentException.class, () ->
analyzer.analyze((CharSequence) null));
+ }
+
+ @Test
+ void testAnalyzeTokensRejectsNullArrays() {
+ final TermAnalyzer analyzer = TermAnalyzer.builder().build();
+ assertThrows(IllegalArgumentException.class,
+ () -> analyzer.analyze(null, new String[] {"NN"}));
+ assertThrows(IllegalArgumentException.class,
+ () -> analyzer.analyze(new String[] {"cat"}, null));
+ }
+
+ @Test
+ void testAnalyzeTokensRejectsNullTokenElement() {
+ final TermAnalyzer analyzer = TermAnalyzer.builder().build();
+ final IllegalArgumentException e =
assertThrows(IllegalArgumentException.class,
+ () -> analyzer.analyze(new String[] {"a", null}, new String[] {"X",
"Y"}));
+ assertTrue(e.getMessage().contains("tokens[1]"), e.getMessage());
+ }
+
+ @Test
+ void testAtRejectsNullDimension() {
+ final Term term = TermAnalyzer.builder().build().analyze("x").get(0);
+ assertThrows(IllegalArgumentException.class, () -> term.at(null));
+ }
+
+ @Test
+ void testBuilderRejectsNullArguments() {
+ assertThrows(IllegalArgumentException.class,
+ () -> TermAnalyzer.builder().whitespace(null));
+ assertThrows(IllegalArgumentException.class,
+ () -> TermAnalyzer.builder().dash(null));
+ assertThrows(IllegalArgumentException.class,
+ () -> TermAnalyzer.builder().caseFold(null));
+ assertThrows(IllegalArgumentException.class,
+ () -> TermAnalyzer.builder().accentFold(null, true));
+ assertThrows(IllegalArgumentException.class, () -> TermAnalyzer.builder()
+ .transform(null, CaseFoldCharSequenceNormalizer.getInstance()));
+ assertThrows(IllegalArgumentException.class,
+ () -> TermAnalyzer.builder().transform(Dimension.CASE_FOLD, null));
+ assertThrows(IllegalArgumentException.class, () ->
TermAnalyzer.builder().stem(null));
+ assertThrows(IllegalArgumentException.class, () ->
TermAnalyzer.builder().lemmatize(null));
+ assertThrows(IllegalArgumentException.class, () ->
TermAnalyzer.builder().tokenizer(null));
+ }
+
+ @Test
+ void testMaxTokenLengthRejectsNonPositiveValues() {
+ assertThrows(IllegalArgumentException.class, () ->
TermAnalyzer.builder().maxTokenLength(0));
+ assertThrows(IllegalArgumentException.class, () ->
TermAnalyzer.builder().maxTokenLength(-1));
+ }
+
+ @Test
+ void testAnalyzeTextWithLemmaConfiguredFailsFastEvenForEmptyText() {
+ final Lemmatizer lemmatizer = new Lemmatizer() {
+ @Override
+ public String[] lemmatize(String[] tokens, String[] tags) {
+ return tokens.clone();
+ }
+
+ @Override
+ public List<List<String>> lemmatize(List<String> tokens, List<String>
tags) {
+ return List.of(tokens);
+ }
+ };
+ final TermAnalyzer analyzer =
TermAnalyzer.builder().lemmatize(lemmatizer).build();
+ // The misconfiguration is reported up front, not only when a token
happens to be produced.
+ assertThrows(IllegalStateException.class, () -> analyzer.analyze(""));
+ }
+
+ @Test
+ void testAtIsThreadSafeUnderConcurrentFirstAccess() throws Exception {
+ // Hammer the lazy cache: many threads request the same unconfigured
dimension of a fresh Term
+ // at the same instant. All of them must observe the same cached value,
with no exceptions from
+ // the concurrent first computation. Repeated over fresh terms to give
races a chance to occur.
+ final int threads = 8;
+ final ExecutorService pool = Executors.newFixedThreadPool(threads);
+ try {
+ final TermAnalyzer analyzer = TermAnalyzer.builder().build();
+ for (int round = 0; round < 50; round++) {
+ final Term term = analyzer.analyze("HELLO").get(0);
+ final CyclicBarrier barrier = new CyclicBarrier(threads);
+ final List<Future<String>> results = new ArrayList<>(threads);
+ for (int i = 0; i < threads; i++) {
+ results.add(pool.submit(() -> {
+ barrier.await();
+ return term.at(Dimension.CASE_FOLD);
+ }));
+ }
+ final String winner = results.get(0).get(10, TimeUnit.SECONDS);
+ assertEquals("hello", winner);
+ for (final Future<String> result : results) {
+ // putIfAbsent keeps exactly one winner; every thread must have
returned that instance.
+ assertSame(winner, result.get(10, TimeUnit.SECONDS));
+ }
+ // And the cache itself holds the same winner.
+ assertSame(winner, term.at(Dimension.CASE_FOLD));
+ }
+ } finally {
+ pool.shutdownNow();
+ }
+ }
+
+ @Test
+ void testConcurrentAccessAcrossDifferentDimensions() throws Exception {
+ // Threads touching different unconfigured dimensions concurrently must
each get the correct
+ // value; the cache is shared but the computations are independent.
+ final Term term = TermAnalyzer.builder().build().analyze("HELLO").get(0);
+ final List<Dimension> dimensions = List.of(
+ Dimension.NFC, Dimension.NFKC, Dimension.WHITESPACE, Dimension.DASH,
Dimension.CASE_FOLD);
+ final ExecutorService pool =
Executors.newFixedThreadPool(dimensions.size());
+ try {
+ final CyclicBarrier barrier = new CyclicBarrier(dimensions.size());
+ final List<Future<String>> results = new ArrayList<>();
+ for (final Dimension dimension : dimensions) {
+ results.add(pool.submit(() -> {
+ barrier.await();
+ return term.at(dimension);
+ }));
+ }
+ assertEquals("HELLO", results.get(0).get(10, TimeUnit.SECONDS)); // NFC:
unchanged
+ assertEquals("hello", results.get(4).get(10, TimeUnit.SECONDS)); //
CASE_FOLD
+ for (final Future<String> result : results) {
+ result.get(10, TimeUnit.SECONDS); // no exceptions anywhere
+ }
+ } finally {
+ pool.shutdownNow();
+ }
+ }
+}
diff --git
a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/TermTest.java
b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/TermTest.java
new file mode 100644
index 000000000..35358aeaf
--- /dev/null
+++
b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/TermTest.java
@@ -0,0 +1,109 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.tools.util.normalizer;
+
+import java.util.List;
+
+import org.junit.jupiter.api.Test;
+
+import opennlp.tools.stemmer.PorterStemmer;
+import opennlp.tools.util.Span;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertSame;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+/**
+ * Exercises the public API of {@link Term}, the layered per-token view
produced by
+ * {@link TermAnalyzer}: {@link Term#original()}, {@link Term#normalized()},
{@link Term#span()},
+ * {@link Term#at(Dimension)}, and {@link Term#peel()}.
+ */
+public class TermTest {
+
+ @Test
+ void originalIsTheSourceOfTruth() {
+ final Term term =
TermAnalyzer.builder().caseFold().build().analyze("Hello").get(0);
+ assertEquals("Hello", term.original());
+ }
+
+ @Test
+ void normalizedIsTheFinalConfiguredDimension() {
+ final Term term =
TermAnalyzer.builder().caseFold().build().analyze("HELLO").get(0);
+ assertEquals("hello", term.normalized());
+ }
+
+ @Test
+ void normalizedEqualsOriginalWhenNoDimensionsConfigured() {
+ final Term term = TermAnalyzer.builder().build().analyze("Hello").get(0);
+ assertEquals("Hello", term.original());
+ assertEquals("Hello", term.normalized());
+ }
+
+ @Test
+ void spanIndexesTheAnalyzedText() {
+ final List<Term> terms = TermAnalyzer.builder().build().analyze("The
Cats");
+ assertEquals(new Span(0, 3), terms.get(0).span());
+ assertEquals(new Span(4, 8), terms.get(1).span());
+ }
+
+ @Test
+ void spanIsNullForPreTokenizedInput() {
+ final Term term = TermAnalyzer.builder().build()
+ .analyze(new String[] {"Cat"}, new String[] {"NN"}).get(0);
+ assertNull(term.span());
+ }
+
+ @Test
+ void atProjectsToAnUnconfiguredDimension() {
+ final Term term = TermAnalyzer.builder().build().analyze("HELLO").get(0);
+ assertEquals("HELLO", term.original());
+ assertEquals("hello", term.at(Dimension.CASE_FOLD));
+ }
+
+ @Test
+ void atIsMemoized() {
+ final Term term = TermAnalyzer.builder().build().analyze("HELLO").get(0);
+ assertSame(term.at(Dimension.CASE_FOLD), term.at(Dimension.CASE_FOLD));
+ }
+
+ @Test
+ void atRejectsNullDimension() {
+ final Term term = TermAnalyzer.builder().build().analyze("x").get(0);
+ assertThrows(IllegalArgumentException.class, () -> term.at(null));
+ }
+
+ @Test
+ void atThrowsWhenDimensionNeedsAMissingEngine() {
+ final Term term = TermAnalyzer.builder().build().analyze("running").get(0);
+ assertThrows(IllegalStateException.class, () -> term.at(Dimension.STEM));
+ }
+
+ @Test
+ void peelReturnsTheLayerBelowTheFinalDimension() {
+ final Term term = TermAnalyzer.builder().caseFold().stem(new
PorterStemmer())
+ .build().analyze("Running").get(0);
+ assertEquals("run", term.normalized()); // STEM is the final dimension
+ assertEquals("running", term.peel()); // the case-folded form, before
stemming
+ }
+
+ @Test
+ void peelEqualsOriginalWithAtMostOneDimension() {
+ final Term term =
TermAnalyzer.builder().caseFold().build().analyze("Hello").get(0);
+ assertEquals(term.original(), term.peel());
+ }
+}
