This is an automated email from the ASF dual-hosted git repository. krickert pushed a commit to branch OPENNLP-1888-DocumentShape in repository https://gitbox.apache.org/repos/asf/opennlp.git
commit 53a15bcb01cd89af0e7db7983b00cf9bd92aeb14 Author: Kristian Rickert <[email protected]> AuthorDate: Thu Jul 16 01:29:09 2026 -0400 OPENNLP-1888: Pipeline example and contract tests for the document container, javadoc precision pass --- .../opennlp/tools/document/ImmutableDocument.java | 3 +- .../main/java/opennlp/tools/document/Layers.java | 2 +- .../tools/document/DocumentContractTest.java | 281 ++++++++++++++++++++ .../document/DocumentPipelineExampleTest.java | 290 +++++++++++++++++++++ .../tools/lemmatizer/LemmatizerAnnotator.java | 5 +- .../opennlp/tools/stemmer/StemmerAnnotator.java | 9 +- 6 files changed, 584 insertions(+), 6 deletions(-) diff --git a/opennlp-api/src/main/java/opennlp/tools/document/ImmutableDocument.java b/opennlp-api/src/main/java/opennlp/tools/document/ImmutableDocument.java index 2ec407ee3..43276b4ce 100644 --- a/opennlp-api/src/main/java/opennlp/tools/document/ImmutableDocument.java +++ b/opennlp-api/src/main/java/opennlp/tools/document/ImmutableDocument.java @@ -69,7 +69,8 @@ final class ImmutableDocument implements Document { if (annotations == null) { return List.of(); } - // safe: with(LayerKey, List) verified every value against the key's type on insertion + // This cast is safe because with(LayerKey, List) verified every value against the + // key's type when the layer was inserted. return (List<Annotation<T>>) (List<?>) annotations; } diff --git a/opennlp-api/src/main/java/opennlp/tools/document/Layers.java b/opennlp-api/src/main/java/opennlp/tools/document/Layers.java index 9ff47cd2c..99f073055 100644 --- a/opennlp-api/src/main/java/opennlp/tools/document/Layers.java +++ b/opennlp-api/src/main/java/opennlp/tools/document/Layers.java @@ -50,6 +50,6 @@ public final class Layers { public static final LayerKey<String> ENTITIES = LayerKey.of("entities", String.class); private Layers() { - // constants only + // This class holds constants only and is never instantiated. } } diff --git a/opennlp-api/src/test/java/opennlp/tools/document/DocumentContractTest.java b/opennlp-api/src/test/java/opennlp/tools/document/DocumentContractTest.java new file mode 100644 index 000000000..c180b1bf9 --- /dev/null +++ b/opennlp-api/src/test/java/opennlp/tools/document/DocumentContractTest.java @@ -0,0 +1,281 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.document; + +import java.util.ArrayList; +import java.util.List; +import java.util.Set; + +import org.junit.jupiter.api.Test; + +import opennlp.tools.util.Span; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertNotSame; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Pins down the observable contract of the document container at its edges: key + * equality, layer ordering, span boundary cases, list immutability, the exact rejection + * messages, and the analyzer's build-time validation. Every expected value in this class + * is asserted exactly so any behavioral drift is caught, not just gross breakage. + */ +public class DocumentContractTest { + + private static final LayerKey<String> WORDS = LayerKey.of("words", String.class); + + /** + * Verifies that two independently created keys with the same id and the same type are + * equal, hash alike, and therefore address the same layer, while remaining distinct + * instances. This is what lets separately compiled producers agree on a layer without + * sharing a constant. + */ + @Test + void testKeysWithSameIdAndTypeAddressTheSameLayer() { + final LayerKey<String> first = LayerKey.of("words", String.class); + final LayerKey<String> second = LayerKey.of("words", String.class); + assertNotSame(first, second); + assertEquals(first, second); + assertEquals(first.hashCode(), second.hashCode()); + assertEquals("words<String>", first.toString()); + + final Document document = Document.of("the") + .with(first, List.of(new Annotation<>(new Span(0, 3), "the"))); + // Reading through the other, equal key yields the very layer added above. + assertEquals(1, document.get(second).size()); + assertEquals("the", document.get(second).get(0).value()); + // A duplicate add through the equal key is rejected like any duplicate. + assertThrows(IllegalArgumentException.class, () -> document.with(second, List.of())); + } + + /** + * Verifies that two keys sharing an id but differing in value type are unequal and + * denote two independent layers that can coexist on one document. + */ + @Test + void testKeysWithSameIdButDifferentTypesAreDifferentLayers() { + final LayerKey<String> asString = LayerKey.of("marks", String.class); + final LayerKey<Integer> asInteger = LayerKey.of("marks", Integer.class); + assertNotEquals(asString, asInteger); + + final Document document = Document.of("ab") + .with(asString, List.of(new Annotation<>(new Span(0, 1), "a"))) + .with(asInteger, List.of(new Annotation<>(new Span(1, 2), 7))); + assertEquals(Set.of(asString, asInteger), document.layers()); + assertEquals("a", document.get(asString).get(0).value()); + assertEquals(7, document.get(asInteger).get(0).value()); + } + + /** + * Verifies that a layer preserves the insertion order of its annotations: the + * container does not sort by span, so a producer that wants span order must supply + * span order. + */ + @Test + void testLayerPreservesInsertionOrder() { + final Document document = Document.of("the dog") + .with(WORDS, List.of( + new Annotation<>(new Span(4, 7), "dog"), + new Annotation<>(new Span(0, 3), "the"))); + final List<Annotation<String>> words = document.get(WORDS); + assertEquals(2, words.size()); + assertEquals(new Span(4, 7), words.get(0).span()); + assertEquals("dog", words.get(0).value()); + assertEquals(new Span(0, 3), words.get(1).span()); + assertEquals("the", words.get(1).value()); + } + + /** + * Verifies that several annotations may share one span within a layer, for example + * alternative readings of the same region, and all of them are retained in order. + */ + @Test + void testAnnotationsWithIdenticalSpansAreAllRetained() { + final Document document = Document.of("bank") + .with(WORDS, List.of( + new Annotation<>(new Span(0, 4), "institution"), + new Annotation<>(new Span(0, 4), "riverside"))); + final List<Annotation<String>> words = document.get(WORDS); + assertEquals(2, words.size()); + assertEquals("institution", words.get(0).value()); + assertEquals("riverside", words.get(1).value()); + assertEquals(words.get(0).span(), words.get(1).span()); + } + + /** + * Verifies that zero-length spans are accepted anywhere within the bounds, including + * at the very end of the text where start and end equal the text length. + */ + @Test + void testZeroLengthSpansAreAccepted() { + final Document document = Document.of("ab") + .with(WORDS, List.of( + new Annotation<>(new Span(1, 1), "between"), + new Annotation<>(new Span(2, 2), "at the end"))); + final List<Annotation<String>> words = document.get(WORDS); + assertEquals(new Span(1, 1), words.get(0).span()); + assertEquals(new Span(2, 2), words.get(1).span()); + assertEquals(0, words.get(0).span().length()); + } + + /** + * Verifies that a span reaching past the end of the text is rejected on insertion + * with a message naming the span, the text length, and the layer. + */ + @Test + void testSpanBeyondTextLengthIsRejectedWithExactMessage() { + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> Document.of("the").with(WORDS, + List.of(new Annotation<>(new Span(0, 4), "the?")))); + assertEquals("span [0..4) exceeds the text length 3 in layer words<String>", + e.getMessage()); + } + + /** + * Verifies that adding a layer under a key that is already present is rejected with a + * message naming the offending layer. + */ + @Test + void testDuplicateLayerIsRejectedWithExactMessage() { + final Document document = Document.of("the").with(WORDS, List.of()); + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> document.with(WORDS, List.of())); + assertEquals("layer is already present: words<String>", e.getMessage()); + } + + /** + * Verifies that reading an absent layer yields an empty, unmodifiable list rather + * than {@code null}, so callers can iterate without a presence check. + */ + @Test + void testAbsentLayerReadsAsUnmodifiableEmptyList() { + final List<Annotation<String>> absent = Document.of("the").get(WORDS); + assertTrue(absent.isEmpty()); + assertThrows(UnsupportedOperationException.class, + () -> absent.add(new Annotation<>(new Span(0, 3), "the"))); + } + + /** + * Verifies that the list returned for a present layer is unmodifiable and detached + * from the caller's input list: mutating the input after the add does not change the + * document. + */ + @Test + void testPresentLayerListIsUnmodifiableAndDetachedFromInput() { + final List<Annotation<String>> input = new ArrayList<>(); + input.add(new Annotation<>(new Span(0, 3), "the")); + final Document document = Document.of("the").with(WORDS, input); + + final List<Annotation<String>> words = document.get(WORDS); + assertThrows(UnsupportedOperationException.class, () -> words.remove(0)); + assertThrows(UnsupportedOperationException.class, + () -> words.add(new Annotation<>(new Span(0, 3), "the"))); + + input.clear(); + assertEquals(1, document.get(WORDS).size()); + } + + /** + * Verifies that the layer key set exposed by a document cannot be mutated by callers. + */ + @Test + void testLayerKeySetIsUnmodifiable() { + final Document document = Document.of("the").with(WORDS, List.of()); + assertThrows(UnsupportedOperationException.class, () -> document.layers().clear()); + } + + /** + * Verifies that an analyzer whose annotator requires a layer no earlier annotator + * provides fails at build time with a message naming the annotator and the missing + * layer. The annotator under test overrides {@code toString()} so the whole message + * can be asserted exactly. + */ + @Test + void testUnsatisfiedRequirementFailsAtBuildTimeWithExactMessage() { + final DocumentAnnotator needsTags = new DocumentAnnotator() { + + @Override + public Document annotate(Document document) { + throw new IllegalStateException("must never run; the pipeline must not build"); + } + + @Override + public Set<LayerKey<?>> requires() { + return Set.of(Layers.POS_TAGS); + } + + @Override + public Set<LayerKey<?>> provides() { + return Set.of(WORDS); + } + + @Override + public String toString() { + return "tag-consumer"; + } + }; + final DocumentAnalyzer.Builder builder = DocumentAnalyzer.builder().add(needsTags); + final IllegalArgumentException e = + assertThrows(IllegalArgumentException.class, builder::build); + assertEquals("annotator tag-consumer requires layer pos<String>," + + " which no earlier annotator provides", e.getMessage()); + } + + /** + * Verifies that building an analyzer without any annotator fails with a message + * stating that a pipeline needs at least one annotator. + */ + @Test + void testEmptyPipelineFailsWithExactMessage() { + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> DocumentAnalyzer.builder().build()); + assertEquals("a pipeline needs at least one annotator", e.getMessage()); + } + + /** + * Verifies that the value type travels through {@link LayerKey}: a layer added under + * an {@code Integer} key reads back as {@code Annotation<Integer>}, so its values + * participate in arithmetic without a cast, and a mismatched value can never enter + * the layer in the first place. + */ + @Test + void testValueTypeTravelsThroughTheKey() { + final LayerKey<Integer> counts = LayerKey.of("counts", Integer.class); + final Document document = Document.of("ab cd") + .with(counts, List.of( + new Annotation<>(new Span(0, 2), 2), + new Annotation<>(new Span(3, 5), 40))); + int sum = 0; + for (final Annotation<Integer> count : document.get(counts)) { + sum += count.value(); + } + assertEquals(42, sum); + + // The insertion-time check backs the typed read: a raw-typed caller cannot place a + // String under the Integer key. + @SuppressWarnings({"unchecked", "rawtypes"}) + final LayerKey<Object> raw = (LayerKey) LayerKey.of("counts2", Integer.class); + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> Document.of("ab").with(raw, + List.of(new Annotation<>(new Span(0, 2), "not a number")))); + assertEquals("value of type java.lang.String does not match layer counts2<Integer>", + e.getMessage()); + } +} diff --git a/opennlp-api/src/test/java/opennlp/tools/document/DocumentPipelineExampleTest.java b/opennlp-api/src/test/java/opennlp/tools/document/DocumentPipelineExampleTest.java new file mode 100644 index 000000000..31a237938 --- /dev/null +++ b/opennlp-api/src/test/java/opennlp/tools/document/DocumentPipelineExampleTest.java @@ -0,0 +1,290 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.document; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.junit.jupiter.api.Test; + +import opennlp.tools.postag.POSTagger; +import opennlp.tools.sentdetect.SentenceDetector; +import opennlp.tools.tokenize.Tokenizer; +import opennlp.tools.util.Sequence; +import opennlp.tools.util.Span; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * Walks through the document pipeline the way a first-time user would: wrap existing + * analysis components in their adapter annotators, add one custom annotator, build a + * {@link DocumentAnalyzer}, analyze a two-sentence text, and read every layer back with + * its spans in original text coordinates. + * + * <p>The wrapped components are tiny deterministic stand-ins defined in this class, so + * every expected span and value below follows directly from the input text. The point + * under demonstration is how the layers connect, not the quality of any single step.</p> + */ +public class DocumentPipelineExampleTest { + + /** + * The key of the custom layer produced by {@link TokenLengthAnnotator}. Any producer + * may introduce such a key in its own code; the container needs no change for it. + */ + private static final LayerKey<Integer> TOKEN_LENGTHS = + LayerKey.of("token-lengths", Integer.class); + + /** + * A deterministic sentence detector that ends a sentence after every period and + * expects a single space between sentences. Only the span-producing method is + * implemented because the adapter calls no other method. + */ + private static final SentenceDetector PERIOD_SPLITTER = new SentenceDetector() { + + @Override + public String[] sentDetect(CharSequence s) { + throw new UnsupportedOperationException("the adapter only calls sentPosDetect"); + } + + @Override + public Span[] sentPosDetect(CharSequence s) { + final String text = s.toString(); + final List<Span> spans = new ArrayList<>(); + int start = 0; + for (int i = 0; i < text.length(); i++) { + if (text.charAt(i) == '.') { + spans.add(new Span(start, i + 1)); + start = i + 2; + } + } + return spans.toArray(new Span[0]); + } + }; + + /** + * A deterministic tokenizer that splits on single space characters and keeps all + * other characters, including sentence-final periods, attached to their token. Only + * the span-producing method is implemented because the adapter calls no other method. + */ + private static final Tokenizer SPACE_TOKENIZER = new Tokenizer() { + + @Override + public String[] tokenize(String s) { + throw new UnsupportedOperationException("the adapter only calls tokenizePos"); + } + + @Override + public Span[] tokenizePos(String s) { + final List<Span> spans = new ArrayList<>(); + int start = -1; + for (int i = 0; i <= s.length(); i++) { + final boolean boundary = i == s.length() || s.charAt(i) == ' '; + if (boundary && start >= 0) { + spans.add(new Span(start, i)); + start = -1; + } else if (!boundary && start < 0) { + start = i; + } + } + return spans.toArray(new Span[0]); + } + }; + + /** + * A deterministic tagger backed by a fixed dictionary covering exactly the tokens of + * the example text. An unknown token fails the test immediately rather than receiving + * a silent fallback tag. + */ + private static final POSTagger DICTIONARY_TAGGER = new POSTagger() { + + private final Map<String, String> tagsByToken = Map.of( + "The", "DT", "dog", "NN", "barks.", "VBZ", "It", "PRP", "naps.", "VBZ"); + + @Override + public String[] tag(String[] sentence) { + final String[] tags = new String[sentence.length]; + for (int i = 0; i < sentence.length; i++) { + final String tag = tagsByToken.get(sentence[i]); + if (tag == null) { + throw new IllegalArgumentException("no tag defined for token: " + sentence[i]); + } + tags[i] = tag; + } + return tags; + } + + @Override + public String[] tag(String[] sentence, Object[] additionalContext) { + return tag(sentence); + } + + @Override + public Sequence[] topKSequences(String[] sentence) { + throw new UnsupportedOperationException("the adapter only calls tag"); + } + + @Override + public Sequence[] topKSequences(String[] sentence, Object[] additionalContext) { + throw new UnsupportedOperationException("the adapter only calls tag"); + } + }; + + /** + * A custom pipeline step written directly against {@link DocumentAnnotator}: it reads + * the token layer and provides {@link #TOKEN_LENGTHS}, one annotation per token on the + * token's span, whose value is the character length of the token text. + */ + private static final class TokenLengthAnnotator implements DocumentAnnotator { + + /** + * Adds the {@link #TOKEN_LENGTHS} layer computed from {@link Layers#TOKENS}. + * + * @param document The document to annotate. Must not be {@code null} and must + * contain the token layer. + * @return A new {@link Document} carrying the token length layer. Never {@code null}. + * @throws IllegalArgumentException Thrown if {@code document} is {@code null} or the + * token layer is absent. + */ + @Override + public Document annotate(Document document) { + if (document == null) { + throw new IllegalArgumentException("document must not be null"); + } + final List<Annotation<String>> tokens = document.get(Layers.TOKENS); + if (tokens.isEmpty()) { + throw new IllegalArgumentException("document lacks the required layer " + + Layers.TOKENS); + } + final List<Annotation<Integer>> lengths = new ArrayList<>(tokens.size()); + for (final Annotation<String> token : tokens) { + lengths.add(new Annotation<>(token.span(), token.value().length())); + } + return document.with(TOKEN_LENGTHS, lengths); + } + + @Override + public Set<LayerKey<?>> requires() { + return Set.of(Layers.TOKENS); + } + + @Override + public Set<LayerKey<?>> provides() { + return Set.of(TOKEN_LENGTHS); + } + } + + /** + * Runs the full pipeline story: sentences, tokens, part-of-speech tags, and one custom + * layer over a two-sentence text, then verifies every annotation of every layer, span + * by span, in original text coordinates. + */ + @Test + void testFullPipelineStory() { + final DocumentAnalyzer analyzer = DocumentAnalyzer.builder() + .add(new SentenceDetectorAnnotator(PERIOD_SPLITTER)) + .add(new TokenizerAnnotator(SPACE_TOKENIZER)) + .add(new POSTaggerAnnotator(DICTIONARY_TAGGER)) + .add(new TokenLengthAnnotator()) + .build(); + + final Document document = analyzer.analyze("The dog barks. It naps."); + + // The document carries exactly the four layers the pipeline provides. + assertEquals(Set.of(Layers.SENTENCES, Layers.TOKENS, Layers.POS_TAGS, TOKEN_LENGTHS), + document.layers()); + assertEquals("The dog barks. It naps.", document.text()); + + // Sentence layer: one annotation per sentence, covering it in document coordinates. + final List<Annotation<String>> sentences = document.get(Layers.SENTENCES); + assertEquals(2, sentences.size()); + assertEquals(new Span(0, 14), sentences.get(0).span()); + assertEquals("The dog barks.", sentences.get(0).value()); + assertEquals(new Span(15, 23), sentences.get(1).span()); + assertEquals("It naps.", sentences.get(1).value()); + + // Token layer: five tokens; the second sentence's spans are shifted back to + // document coordinates, so every span can index into the original text. + final List<Annotation<String>> tokens = document.get(Layers.TOKENS); + assertEquals(5, tokens.size()); + assertEquals(new Span(0, 3), tokens.get(0).span()); + assertEquals("The", tokens.get(0).value()); + assertEquals(new Span(4, 7), tokens.get(1).span()); + assertEquals("dog", tokens.get(1).value()); + assertEquals(new Span(8, 14), tokens.get(2).span()); + assertEquals("barks.", tokens.get(2).value()); + assertEquals(new Span(15, 17), tokens.get(3).span()); + assertEquals("It", tokens.get(3).value()); + assertEquals(new Span(18, 23), tokens.get(4).span()); + assertEquals("naps.", tokens.get(4).value()); + + // Tag layer: aligned with the token layer by position, each tag on its token's span. + final List<Annotation<String>> tags = document.get(Layers.POS_TAGS); + assertEquals(5, tags.size()); + assertEquals("DT", tags.get(0).value()); + assertEquals("NN", tags.get(1).value()); + assertEquals("VBZ", tags.get(2).value()); + assertEquals("PRP", tags.get(3).value()); + assertEquals("VBZ", tags.get(4).value()); + for (int i = 0; i < tags.size(); i++) { + assertEquals(tokens.get(i).span(), tags.get(i).span()); + } + + // Custom layer: the container returns it as List<Annotation<Integer>>, so the + // values are used as numbers without a cast. + final List<Annotation<Integer>> lengths = document.get(TOKEN_LENGTHS); + assertEquals(5, lengths.size()); + assertEquals(3, lengths.get(0).value()); + assertEquals(3, lengths.get(1).value()); + assertEquals(6, lengths.get(2).value()); + assertEquals(2, lengths.get(3).value()); + assertEquals(5, lengths.get(4).value()); + for (int i = 0; i < lengths.size(); i++) { + assertEquals(tokens.get(i).span(), lengths.get(i).span()); + } + + // Every span refers to the original text, so covered text always round-trips. + for (final Annotation<String> token : tokens) { + assertEquals(token.value(), + token.span().getCoveredText(document.text()).toString()); + } + } + + /** + * Verifies that the analyzer leaves the input untouched between calls: analyzing two + * texts with the same analyzer yields two independent documents. + */ + @Test + void testAnalyzerIsReusableAcrossTexts() { + final DocumentAnalyzer analyzer = DocumentAnalyzer.builder() + .add(new SentenceDetectorAnnotator(PERIOD_SPLITTER)) + .add(new TokenizerAnnotator(SPACE_TOKENIZER)) + .build(); + + final Document first = analyzer.analyze("The dog barks."); + final Document second = analyzer.analyze("It naps."); + + assertEquals(3, first.get(Layers.TOKENS).size()); + assertEquals(2, second.get(Layers.TOKENS).size()); + assertEquals("The dog barks.", first.text()); + assertEquals("It naps.", second.text()); + assertEquals(1, first.get(Layers.SENTENCES).size()); + assertEquals(1, second.get(Layers.SENTENCES).size()); + } +} diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/lemmatizer/LemmatizerAnnotator.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/lemmatizer/LemmatizerAnnotator.java index f1733da4e..37fc3478a 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/lemmatizer/LemmatizerAnnotator.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/lemmatizer/LemmatizerAnnotator.java @@ -35,7 +35,10 @@ import opennlp.tools.document.Layers; */ public class LemmatizerAnnotator implements DocumentAnnotator { - /** Lemmas; aligned with the token layer, each annotation on its token's span. */ + /** + * The lemma layer. It is aligned with the token layer by position, and each annotation + * carries the lemma of its token on that token's span. + */ public static final LayerKey<String> LEMMAS = LayerKey.of("lemmas", String.class); private final Lemmatizer lemmatizer; diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/StemmerAnnotator.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/StemmerAnnotator.java index da3b109a6..8d76d018a 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/StemmerAnnotator.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/StemmerAnnotator.java @@ -31,14 +31,17 @@ import opennlp.tools.document.Layers; * Adapts a {@link Stemmer} to the document pipeline: stems the token layer and provides * {@link #STEMS}, one annotation per token on the token's span. * - * <p>Stemming needs no tags, so unlike lemmatization this annotator only requires the - * token layer.</p> + * <p>Stemming operates on the token surface alone, so this annotator requires only the + * token layer; no part-of-speech tags are involved.</p> * * @since 3.0.0 */ public class StemmerAnnotator implements DocumentAnnotator { - /** Stems; aligned with the token layer, each annotation on its token's span. */ + /** + * The stem layer. It is aligned with the token layer by position, and each annotation + * carries the stem of its token on that token's span. + */ public static final LayerKey<String> STEMS = LayerKey.of("stems", String.class); private final Stemmer stemmer;
