This is an automated email from the ASF dual-hosted git repository. krickert pushed a commit to branch OPENNLP-1888-DocumentShape in repository https://gitbox.apache.org/repos/asf/opennlp.git
commit eec278db5f4d65f9adf85e45957d05cf2b9b9ee5 Author: Kristian Rickert <[email protected]> AuthorDate: Wed Jul 15 09:00:07 2026 -0400 OPENNLP-1888: Lemma and stem layer adapters --- .../tools/lemmatizer/LemmatizerAnnotator.java | 94 ++++++++++++++++++++++ .../opennlp/tools/stemmer/StemmerAnnotator.java | 81 +++++++++++++++++++ .../tools/lemmatizer/LemmatizerAnnotatorTest.java | 82 +++++++++++++++++++ .../tools/stemmer/StemmerAnnotatorTest.java | 56 +++++++++++++ 4 files changed, 313 insertions(+) diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/lemmatizer/LemmatizerAnnotator.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/lemmatizer/LemmatizerAnnotator.java new file mode 100644 index 000000000..f1733da4e --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/lemmatizer/LemmatizerAnnotator.java @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.lemmatizer; + +import java.util.ArrayList; +import java.util.List; +import java.util.Set; + +import opennlp.tools.document.Annotation; +import opennlp.tools.document.Document; +import opennlp.tools.document.DocumentAnnotator; +import opennlp.tools.document.LayerKey; +import opennlp.tools.document.Layers; + +/** + * Adapts a {@link Lemmatizer} to the document pipeline: lemmatizes the token layer with + * its tags and provides {@link #LEMMAS}, one annotation per token on the token's span. + * + * @since 3.0.0 + */ +public class LemmatizerAnnotator implements DocumentAnnotator { + + /** Lemmas; aligned with the token layer, each annotation on its token's span. */ + public static final LayerKey<String> LEMMAS = LayerKey.of("lemmas", String.class); + + private final Lemmatizer lemmatizer; + + /** + * Initializes the adapter. + * + * @param lemmatizer The lemmatizer to delegate to. Must not be {@code null}. + * @throws IllegalArgumentException Thrown if {@code lemmatizer} is {@code null}. + */ + public LemmatizerAnnotator(Lemmatizer lemmatizer) { + if (lemmatizer == null) { + throw new IllegalArgumentException("lemmatizer must not be null"); + } + this.lemmatizer = lemmatizer; + } + + @Override + public Document annotate(Document document) { + if (document == null) { + throw new IllegalArgumentException("document must not be null"); + } + final List<Annotation<String>> tokens = document.get(Layers.TOKENS); + final List<Annotation<String>> tags = document.get(Layers.POS_TAGS); + if (tags.size() != tokens.size()) { + throw new IllegalArgumentException("document needs aligned " + + Layers.TOKENS + " and " + Layers.POS_TAGS + " layers"); + } + final String[] words = new String[tokens.size()]; + final String[] posTags = new String[tokens.size()]; + for (int i = 0; i < words.length; i++) { + words[i] = tokens.get(i).value(); + posTags[i] = tags.get(i).value(); + } + final String[] lemmas = lemmatizer.lemmatize(words, posTags); + if (lemmas.length != words.length) { + throw new IllegalArgumentException( + "lemmatizer returned " + lemmas.length + " lemmas for " + words.length + " tokens"); + } + final List<Annotation<String>> layer = new ArrayList<>(lemmas.length); + for (int i = 0; i < lemmas.length; i++) { + layer.add(new Annotation<>(tokens.get(i).span(), lemmas[i])); + } + return document.with(LEMMAS, layer); + } + + @Override + public Set<LayerKey<?>> requires() { + return Set.of(Layers.TOKENS, Layers.POS_TAGS); + } + + @Override + public Set<LayerKey<?>> provides() { + return Set.of(LEMMAS); + } +} diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/StemmerAnnotator.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/StemmerAnnotator.java new file mode 100644 index 000000000..da3b109a6 --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/StemmerAnnotator.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.stemmer; + +import java.util.ArrayList; +import java.util.List; +import java.util.Set; + +import opennlp.tools.document.Annotation; +import opennlp.tools.document.Document; +import opennlp.tools.document.DocumentAnnotator; +import opennlp.tools.document.LayerKey; +import opennlp.tools.document.Layers; + +/** + * Adapts a {@link Stemmer} to the document pipeline: stems the token layer and provides + * {@link #STEMS}, one annotation per token on the token's span. + * + * <p>Stemming needs no tags, so unlike lemmatization this annotator only requires the + * token layer.</p> + * + * @since 3.0.0 + */ +public class StemmerAnnotator implements DocumentAnnotator { + + /** Stems; aligned with the token layer, each annotation on its token's span. */ + public static final LayerKey<String> STEMS = LayerKey.of("stems", String.class); + + private final Stemmer stemmer; + + /** + * Initializes the adapter. + * + * @param stemmer The stemmer to delegate to. Must not be {@code null}. + * @throws IllegalArgumentException Thrown if {@code stemmer} is {@code null}. + */ + public StemmerAnnotator(Stemmer stemmer) { + if (stemmer == null) { + throw new IllegalArgumentException("stemmer must not be null"); + } + this.stemmer = stemmer; + } + + @Override + public Document annotate(Document document) { + if (document == null) { + throw new IllegalArgumentException("document must not be null"); + } + final List<Annotation<String>> tokens = document.get(Layers.TOKENS); + final List<Annotation<String>> layer = new ArrayList<>(tokens.size()); + for (final Annotation<String> token : tokens) { + layer.add(new Annotation<>(token.span(), stemmer.stem(token.value()).toString())); + } + return document.with(STEMS, layer); + } + + @Override + public Set<LayerKey<?>> requires() { + return Set.of(Layers.TOKENS); + } + + @Override + public Set<LayerKey<?>> provides() { + return Set.of(STEMS); + } +} diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/lemmatizer/LemmatizerAnnotatorTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/lemmatizer/LemmatizerAnnotatorTest.java new file mode 100644 index 000000000..6b2fb8b8d --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/lemmatizer/LemmatizerAnnotatorTest.java @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.lemmatizer; + +import java.util.List; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import opennlp.tools.document.Annotation; +import opennlp.tools.document.Document; +import opennlp.tools.document.Layers; +import opennlp.tools.util.Span; + +public class LemmatizerAnnotatorTest { + + /** Lowercases verbs and keeps everything else, enough to observe the adapter. */ + private static final Lemmatizer FIXTURE = new Lemmatizer() { + @Override + public String[] lemmatize(String[] toks, String[] tags) { + final String[] lemmas = new String[toks.length]; + for (int i = 0; i < toks.length; i++) { + lemmas[i] = "VERB".equals(tags[i]) ? "run" : toks[i]; + } + return lemmas; + } + + @Override + public List<List<String>> lemmatize(List<String> toks, List<String> tags) { + throw new UnsupportedOperationException(); + } + }; + + @Test + void testLemmasAlignWithTokens() { + final Document document = Document.of("She ran home") + .with(Layers.TOKENS, List.of( + new Annotation<>(new Span(0, 3), "She"), + new Annotation<>(new Span(4, 7), "ran"), + new Annotation<>(new Span(8, 12), "home"))) + .with(Layers.POS_TAGS, List.of( + new Annotation<>(new Span(0, 3), "PRON"), + new Annotation<>(new Span(4, 7), "VERB"), + new Annotation<>(new Span(8, 12), "NOUN"))); + + final Document lemmatized = new LemmatizerAnnotator(FIXTURE).annotate(document); + + final List<Annotation<String>> lemmas = lemmatized.get(LemmatizerAnnotator.LEMMAS); + Assertions.assertEquals(3, lemmas.size()); + Assertions.assertEquals("run", lemmas.get(1).value()); + Assertions.assertEquals(new Span(4, 7), lemmas.get(1).span()); + Assertions.assertEquals("home", lemmas.get(2).value()); + } + + @Test + void testInvalidArguments() { + Assertions.assertThrows(IllegalArgumentException.class, + () -> new LemmatizerAnnotator(null)); + final LemmatizerAnnotator annotator = new LemmatizerAnnotator(FIXTURE); + Assertions.assertThrows(IllegalArgumentException.class, () -> annotator.annotate(null)); + final Document misaligned = Document.of("a b") + .with(Layers.TOKENS, List.of(new Annotation<>(new Span(0, 1), "a"))) + .with(Layers.POS_TAGS, List.of()); + Assertions.assertThrows(IllegalArgumentException.class, + () -> annotator.annotate(misaligned)); + } +} diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/StemmerAnnotatorTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/StemmerAnnotatorTest.java new file mode 100644 index 000000000..931b320e3 --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/StemmerAnnotatorTest.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.stemmer; + +import java.util.List; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import opennlp.tools.document.Annotation; +import opennlp.tools.document.Document; +import opennlp.tools.document.Layers; +import opennlp.tools.util.Span; + +public class StemmerAnnotatorTest { + + @Test + void testStemsAlignWithTokens() { + final Document document = Document.of("running dogs") + .with(Layers.TOKENS, List.of( + new Annotation<>(new Span(0, 7), "running"), + new Annotation<>(new Span(8, 12), "dogs"))); + + final Document stemmed = new StemmerAnnotator( + new PorterStemmer()).annotate(document); + + final List<Annotation<String>> stems = stemmed.get(StemmerAnnotator.STEMS); + Assertions.assertEquals(2, stems.size()); + Assertions.assertEquals("run", stems.get(0).value()); + Assertions.assertEquals(new Span(0, 7), stems.get(0).span()); + Assertions.assertEquals("dog", stems.get(1).value()); + } + + @Test + void testInvalidArguments() { + Assertions.assertThrows(IllegalArgumentException.class, + () -> new StemmerAnnotator(null)); + final StemmerAnnotator annotator = new StemmerAnnotator(new PorterStemmer()); + Assertions.assertThrows(IllegalArgumentException.class, () -> annotator.annotate(null)); + } +}
