This is an automated email from the ASF dual-hosted git repository. krickert pushed a commit to branch OPENNLP-1888-DocumentShape in repository https://gitbox.apache.org/repos/asf/opennlp.git
commit 29149f4d4661b5a37409434b8502683c8a134771 Author: Kristian Rickert <[email protected]> AuthorDate: Tue Jul 14 21:27:22 2026 -0400 OPENNLP-1888: Document annotation container: typed offset-anchored layers over the original text Adds opennlp.tools.document to opennlp-api: Document (immutable, copy-on-add layer container over the original text), Annotation (a typed value on a Span), LayerKey (open, typed layer identity), and DocumentAnnotator (pipeline step declaring the layers it requires and provides). DocumentAnalyzer assembles annotators into a pipeline validated at build time. Standard keys in Layers cover sentences, tokens, part-of-speech tags, and entities, populated through thin adapters over the existing SentenceDetector, Tokenizer, POSTagger, and TokenNameFinder interfaces, which stay the primary API for single-task use and are unchanged. All spans refer to the text as supplied. No new dependencies. --- .../java/opennlp/tools/document/Annotation.java | 55 ++++++ .../main/java/opennlp/tools/document/Document.java | 87 +++++++++ .../opennlp/tools/document/DocumentAnalyzer.java | 116 ++++++++++++ .../opennlp/tools/document/DocumentAnnotator.java | 59 +++++++ .../opennlp/tools/document/ImmutableDocument.java | 107 ++++++++++++ .../main/java/opennlp/tools/document/LayerKey.java | 102 +++++++++++ .../main/java/opennlp/tools/document/Layers.java | 55 ++++++ .../tools/document/NameFinderAnnotator.java | 88 ++++++++++ .../opennlp/tools/document/POSTaggerAnnotator.java | 80 +++++++++ .../tools/document/SentenceDetectorAnnotator.java | 70 ++++++++ .../opennlp/tools/document/TokenizerAnnotator.java | 84 +++++++++ .../tools/document/DocumentAnalyzerTest.java | 194 +++++++++++++++++++++ .../java/opennlp/tools/document/DocumentTest.java | 125 +++++++++++++ .../tools/document/NameFinderAnnotatorTest.java | 87 +++++++++ 14 files changed, 1309 insertions(+) diff --git a/opennlp-api/src/main/java/opennlp/tools/document/Annotation.java b/opennlp-api/src/main/java/opennlp/tools/document/Annotation.java new file mode 100644 index 000000000..eaed6bf7f --- /dev/null +++ b/opennlp-api/src/main/java/opennlp/tools/document/Annotation.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.document; + +import opennlp.tools.util.Span; + +/** + * One annotation of a {@link Document}: a typed value anchored to a {@link Span} of the + * document's original text. + * + * <p>The span always refers to the text the document was created with, never to a + * normalized or otherwise derived form, so any annotation can be highlighted in what the + * caller supplied. Annotations that need to reference other annotations, for example a + * dependency arc naming its head token, do so by the index of the target annotation + * within its layer, never by object identity.</p> + * + * @param span The location of the annotation in the original text. Must not be + * {@code null}. + * @param value The annotation value. Must not be {@code null}. + * @param <T> The type of the annotation value. + * + * @since 3.0.0 + */ +public record Annotation<T>(Span span, T value) { + + /** + * Validates the annotation. + * + * @throws IllegalArgumentException Thrown if {@code span} or {@code value} is + * {@code null}. + */ + public Annotation { + if (span == null) { + throw new IllegalArgumentException("span must not be null"); + } + if (value == null) { + throw new IllegalArgumentException("value must not be null"); + } + } +} diff --git a/opennlp-api/src/main/java/opennlp/tools/document/Document.java b/opennlp-api/src/main/java/opennlp/tools/document/Document.java new file mode 100644 index 000000000..53453ff85 --- /dev/null +++ b/opennlp-api/src/main/java/opennlp/tools/document/Document.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.document; + +import java.util.List; +import java.util.Set; + +/** + * An immutable, offset-anchored annotation container: the original text of one document + * plus any number of typed annotation layers over it. + * + * <p>A layer is a list of {@link Annotation annotations} identified by a + * {@link LayerKey}. The container itself knows nothing about specific layers; every + * analysis capability contributes its results as one more layer without any change to + * this interface, which is what keeps new capabilities additive. All spans refer to + * {@link #text()} as supplied, never to a derived form.</p> + * + * <p>Documents are immutable: {@link #with(LayerKey, List)} returns a new document that + * shares the unchanged layers. Instances are safe to share between threads.</p> + * + * @since 3.0.0 + */ +public interface Document { + + /** + * Creates an empty {@link Document} over a text. + * + * @param text The original document text. Must not be {@code null}. + * @return A {@link Document} without any layers. Never {@code null}. + * @throws IllegalArgumentException Thrown if {@code text} is {@code null}. + */ + static Document of(CharSequence text) { + return ImmutableDocument.empty(text); + } + + /** + * @return The original text of the document. Never {@code null}. + */ + CharSequence text(); + + /** + * Retrieves the annotations of one layer. + * + * @param layer The layer to read. Must not be {@code null}. + * @param <T> The type of the layer's annotation values. + * @return The layer's annotations in their layer order, or an empty list when the + * layer is absent. Never {@code null}; the list is unmodifiable. + * @throws IllegalArgumentException Thrown if {@code layer} is {@code null}. + */ + <T> List<Annotation<T>> get(LayerKey<T> layer); + + /** + * @return The keys of all layers present on the document. Never {@code null}; the set + * is unmodifiable. + */ + Set<LayerKey<?>> layers(); + + /** + * Returns a new document with one layer added. + * + * @param layer The key of the layer to add. Must not be {@code null} and must not + * already be present. + * @param annotations The annotations of the layer. Must not be {@code null}, must not + * contain {@code null}, every value must be assignable to the + * layer's type, and every span must lie within the text bounds. + * @param <T> The type of the layer's annotation values. + * @return A new {@link Document} sharing this document's text and existing layers. + * Never {@code null}. + * @throws IllegalArgumentException Thrown if any of the above constraints is violated. + */ + <T> Document with(LayerKey<T> layer, List<Annotation<T>> annotations); +} diff --git a/opennlp-api/src/main/java/opennlp/tools/document/DocumentAnalyzer.java b/opennlp-api/src/main/java/opennlp/tools/document/DocumentAnalyzer.java new file mode 100644 index 000000000..4ca353ca1 --- /dev/null +++ b/opennlp-api/src/main/java/opennlp/tools/document/DocumentAnalyzer.java @@ -0,0 +1,116 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.document; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +/** + * Runs a fixed sequence of {@link DocumentAnnotator annotators} over a text, producing + * one {@link Document} that carries every step's layers. + * + * <p>The pipeline is validated at build time: every annotator's required layers must be + * provided by an earlier annotator, so a misordered pipeline fails when it is assembled + * rather than midway through a document. The analyzer holds no per-call state; it is as + * thread-safe as the annotators it is built from.</p> + * + * @since 3.0.0 + */ +public final class DocumentAnalyzer { + + private final List<DocumentAnnotator> annotators; + + private DocumentAnalyzer(List<DocumentAnnotator> annotators) { + this.annotators = annotators; + } + + /** + * @return A new {@link Builder}. Never {@code null}. + */ + public static Builder builder() { + return new Builder(); + } + + /** + * Analyzes a text by running every annotator in order. + * + * @param text The original document text. Must not be {@code null}. + * @return The annotated {@link Document}. Never {@code null}. + * @throws IllegalArgumentException Thrown if {@code text} is {@code null}. + */ + public Document analyze(CharSequence text) { + Document document = Document.of(text); + for (final DocumentAnnotator annotator : annotators) { + document = annotator.annotate(document); + } + return document; + } + + /** + * Assembles a {@link DocumentAnalyzer} from annotators in execution order. + */ + public static final class Builder { + + private final List<DocumentAnnotator> annotators = new ArrayList<>(); + + private Builder() { + } + + /** + * Appends an annotator to the pipeline. + * + * @param annotator The annotator to run after the ones already added. Must not be + * {@code null}. + * @return This {@link Builder}. Never {@code null}. + * @throws IllegalArgumentException Thrown if {@code annotator} is {@code null}. + */ + public Builder add(DocumentAnnotator annotator) { + if (annotator == null) { + throw new IllegalArgumentException("annotator must not be null"); + } + annotators.add(annotator); + return this; + } + + /** + * Validates the pipeline and builds the analyzer. + * + * @return A {@link DocumentAnalyzer}. Never {@code null}. + * @throws IllegalArgumentException Thrown if the pipeline is empty or an annotator + * requires a layer no earlier annotator provides. + */ + public DocumentAnalyzer build() { + if (annotators.isEmpty()) { + throw new IllegalArgumentException("a pipeline needs at least one annotator"); + } + final Set<LayerKey<?>> available = new HashSet<>(); + for (final DocumentAnnotator annotator : annotators) { + for (final LayerKey<?> required : annotator.requires()) { + if (!available.contains(required)) { + throw new IllegalArgumentException("annotator " + annotator + + " requires layer " + required + ", which no earlier annotator provides"); + } + } + available.addAll(annotator.provides()); + } + return new DocumentAnalyzer(List.copyOf(annotators)); + } + } +} diff --git a/opennlp-api/src/main/java/opennlp/tools/document/DocumentAnnotator.java b/opennlp-api/src/main/java/opennlp/tools/document/DocumentAnnotator.java new file mode 100644 index 000000000..9ea2c65a4 --- /dev/null +++ b/opennlp-api/src/main/java/opennlp/tools/document/DocumentAnnotator.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.document; + +import java.util.Set; + +/** + * A pipeline step that reads layers from a {@link Document} and returns a new document + * with its own layers added. + * + * <p>An annotator declares the layers it {@link #requires()} and {@link #provides()}, so + * a {@link DocumentAnalyzer} can validate a pipeline before running it. Annotators are + * usually thin adapters over an existing analysis component and should hold no per-call + * state, so one instance can serve concurrent pipelines when the wrapped component + * allows it.</p> + * + * @since 3.0.0 + */ +public interface DocumentAnnotator { + + /** + * Annotates a document. + * + * @param document The document to annotate. Must not be {@code null} and must contain + * every layer named by {@link #requires()}. + * @return A new {@link Document} carrying the layers named by {@link #provides()} in + * addition to the input layers. Never {@code null}. + * @throws IllegalArgumentException Thrown if {@code document} is {@code null} or lacks + * a required layer. + */ + Document annotate(Document document); + + /** + * @return The keys of the layers this annotator reads. Never {@code null}. + */ + default Set<LayerKey<?>> requires() { + return Set.of(); + } + + /** + * @return The keys of the layers this annotator adds. Never {@code null}. + */ + Set<LayerKey<?>> provides(); +} diff --git a/opennlp-api/src/main/java/opennlp/tools/document/ImmutableDocument.java b/opennlp-api/src/main/java/opennlp/tools/document/ImmutableDocument.java new file mode 100644 index 000000000..2ec407ee3 --- /dev/null +++ b/opennlp-api/src/main/java/opennlp/tools/document/ImmutableDocument.java @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.document; + +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import opennlp.tools.util.Span; + +/** + * The default {@link Document} implementation: an unmodifiable map from layer key to an + * unmodifiable annotation list. Adding a layer copies the map, not the layers, so + * documents grown from a common ancestor share their layer lists. + */ +final class ImmutableDocument implements Document { + + private final CharSequence text; + private final Map<LayerKey<?>, List<Annotation<?>>> layers; + + private ImmutableDocument(CharSequence text, Map<LayerKey<?>, List<Annotation<?>>> layers) { + this.text = text; + this.layers = layers; + } + + /** + * Creates a document without any layers. + * + * @param text The original document text. Must not be {@code null}. + * @return An empty {@link ImmutableDocument}. Never {@code null}. + * @throws IllegalArgumentException Thrown if {@code text} is {@code null}. + */ + static ImmutableDocument empty(CharSequence text) { + if (text == null) { + throw new IllegalArgumentException("text must not be null"); + } + return new ImmutableDocument(text, Collections.emptyMap()); + } + + @Override + public CharSequence text() { + return text; + } + + @Override + @SuppressWarnings("unchecked") + public <T> List<Annotation<T>> get(LayerKey<T> layer) { + if (layer == null) { + throw new IllegalArgumentException("layer must not be null"); + } + final List<Annotation<?>> annotations = layers.get(layer); + if (annotations == null) { + return List.of(); + } + // safe: with(LayerKey, List) verified every value against the key's type on insertion + return (List<Annotation<T>>) (List<?>) annotations; + } + + @Override + public Set<LayerKey<?>> layers() { + return Collections.unmodifiableSet(layers.keySet()); + } + + @Override + public <T> Document with(LayerKey<T> layer, List<Annotation<T>> annotations) { + if (layer == null || annotations == null) { + throw new IllegalArgumentException("layer and annotations must not be null"); + } + if (layers.containsKey(layer)) { + throw new IllegalArgumentException("layer is already present: " + layer); + } + for (final Annotation<T> annotation : annotations) { + if (annotation == null) { + throw new IllegalArgumentException("annotations must not contain null: " + layer); + } + if (!layer.type().isInstance(annotation.value())) { + throw new IllegalArgumentException("value of type " + + annotation.value().getClass().getName() + " does not match layer " + layer); + } + final Span span = annotation.span(); + if (span.getEnd() > text.length()) { + throw new IllegalArgumentException("span " + span + " exceeds the text length " + + text.length() + " in layer " + layer); + } + } + final Map<LayerKey<?>, List<Annotation<?>>> grown = new LinkedHashMap<>(layers); + grown.put(layer, List.copyOf(annotations)); + return new ImmutableDocument(text, Collections.unmodifiableMap(grown)); + } +} diff --git a/opennlp-api/src/main/java/opennlp/tools/document/LayerKey.java b/opennlp-api/src/main/java/opennlp/tools/document/LayerKey.java new file mode 100644 index 000000000..458a61b4c --- /dev/null +++ b/opennlp-api/src/main/java/opennlp/tools/document/LayerKey.java @@ -0,0 +1,102 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.document; + +import java.util.Objects; + +/** + * Identifies one annotation layer of a {@link Document} and carries the type of that + * layer's annotation values, so reading a layer back is statically typed. + * + * <p>The key space is deliberately open: any producer may define new keys in its own + * package, and the container never enumerates them. Two keys are equal when both their + * id and their value type are equal, so independently created constants for the same + * layer interoperate. Standard keys for the toolkit's own results live in + * {@link Layers}.</p> + * + * @param <T> The type of the annotation values stored under this key. + * + * @since 3.0.0 + */ +public final class LayerKey<T> { + + private final String id; + private final Class<T> type; + + private LayerKey(String id, Class<T> type) { + this.id = id; + this.type = type; + } + + /** + * Creates a {@link LayerKey}. + * + * @param id The layer identifier, for example {@code tokens}. Must not be {@code null} + * or blank. + * @param type The class of the annotation values stored under the key. Must not be + * {@code null}. + * @param <T> The type of the annotation values. + * @return A {@link LayerKey}. Never {@code null}. + * @throws IllegalArgumentException Thrown if {@code id} is {@code null} or blank, or + * {@code type} is {@code null}. + */ + public static <T> LayerKey<T> of(String id, Class<T> type) { + if (id == null || id.isBlank()) { + throw new IllegalArgumentException("id must not be null or blank"); + } + if (type == null) { + throw new IllegalArgumentException("type must not be null"); + } + return new LayerKey<>(id, type); + } + + /** + * @return The layer identifier. Never {@code null}. + */ + public String id() { + return id; + } + + /** + * @return The class of the annotation values stored under this key. Never {@code null}. + */ + public Class<T> type() { + return type; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (!(obj instanceof LayerKey<?> other)) { + return false; + } + return id.equals(other.id) && type.equals(other.type); + } + + @Override + public int hashCode() { + return Objects.hash(id, type); + } + + @Override + public String toString() { + return id + '<' + type.getSimpleName() + '>'; + } +} diff --git a/opennlp-api/src/main/java/opennlp/tools/document/Layers.java b/opennlp-api/src/main/java/opennlp/tools/document/Layers.java new file mode 100644 index 000000000..9ff47cd2c --- /dev/null +++ b/opennlp-api/src/main/java/opennlp/tools/document/Layers.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.document; + +/** + * The standard {@link LayerKey layer keys} for the results the toolkit produces itself. + * + * <p>This class is a convenience, not a registry: the key space stays open, and any + * producer may define further keys in its own package. New capabilities must never + * require an addition here to function.</p> + * + * @since 3.0.0 + */ +public final class Layers { + + /** + * Sentence boundaries; each annotation covers one sentence and carries its text. + */ + public static final LayerKey<String> SENTENCES = LayerKey.of("sentences", String.class); + + /** + * Token boundaries; each annotation covers one token and carries its text. + */ + public static final LayerKey<String> TOKENS = LayerKey.of("tokens", String.class); + + /** + * Part-of-speech tags; one annotation per token, aligned with {@link #TOKENS} by + * position, carrying the tag. + */ + public static final LayerKey<String> POS_TAGS = LayerKey.of("pos", String.class); + + /** + * Named entities; each annotation covers one mention and carries the entity type. + */ + public static final LayerKey<String> ENTITIES = LayerKey.of("entities", String.class); + + private Layers() { + // constants only + } +} diff --git a/opennlp-api/src/main/java/opennlp/tools/document/NameFinderAnnotator.java b/opennlp-api/src/main/java/opennlp/tools/document/NameFinderAnnotator.java new file mode 100644 index 000000000..f5bfecdaf --- /dev/null +++ b/opennlp-api/src/main/java/opennlp/tools/document/NameFinderAnnotator.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.document; + +import java.util.ArrayList; +import java.util.List; +import java.util.Set; + +import opennlp.tools.namefind.TokenNameFinder; +import opennlp.tools.util.Span; + +/** + * Adapts a {@link TokenNameFinder} to the document pipeline: reads {@link Layers#TOKENS}, + * maps the finder's token-index spans to character spans on the original text, and + * provides {@link Layers#ENTITIES} carrying the entity type. + * + * <p>The finder's adaptive data is cleared after each document, so document order does + * not leak between pipeline runs.</p> + * + * @since 3.0.0 + */ +public class NameFinderAnnotator implements DocumentAnnotator { + + private final TokenNameFinder finder; + + /** + * Initializes the adapter. + * + * @param finder The name finder to delegate to. Must not be {@code null}. + * @throws IllegalArgumentException Thrown if {@code finder} is {@code null}. + */ + public NameFinderAnnotator(TokenNameFinder finder) { + if (finder == null) { + throw new IllegalArgumentException("finder must not be null"); + } + this.finder = finder; + } + + @Override + public Document annotate(Document document) { + if (document == null) { + throw new IllegalArgumentException("document must not be null"); + } + final List<Annotation<String>> tokens = document.get(Layers.TOKENS); + if (tokens.isEmpty()) { + throw new IllegalArgumentException("document lacks the required layer " + + Layers.TOKENS); + } + final String[] words = new String[tokens.size()]; + for (int i = 0; i < words.length; i++) { + words[i] = tokens.get(i).value(); + } + final List<Annotation<String>> entities = new ArrayList<>(); + for (final Span mention : finder.find(words)) { + final int start = tokens.get(mention.getStart()).span().getStart(); + final int end = tokens.get(mention.getEnd() - 1).span().getEnd(); + final String type = mention.getType() == null ? "default" : mention.getType(); + entities.add(new Annotation<>(new Span(start, end, type), type)); + } + finder.clearAdaptiveData(); + return document.with(Layers.ENTITIES, entities); + } + + @Override + public Set<LayerKey<?>> requires() { + return Set.of(Layers.TOKENS); + } + + @Override + public Set<LayerKey<?>> provides() { + return Set.of(Layers.ENTITIES); + } +} diff --git a/opennlp-api/src/main/java/opennlp/tools/document/POSTaggerAnnotator.java b/opennlp-api/src/main/java/opennlp/tools/document/POSTaggerAnnotator.java new file mode 100644 index 000000000..1234b6803 --- /dev/null +++ b/opennlp-api/src/main/java/opennlp/tools/document/POSTaggerAnnotator.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.document; + +import java.util.ArrayList; +import java.util.List; +import java.util.Set; + +import opennlp.tools.postag.POSTagger; + +/** + * Adapts a {@link POSTagger} to the document pipeline: reads {@link Layers#TOKENS} and + * provides {@link Layers#POS_TAGS}, one tag annotation per token on the token's span. + * + * @since 3.0.0 + */ +public class POSTaggerAnnotator implements DocumentAnnotator { + + private final POSTagger tagger; + + /** + * Initializes the adapter. + * + * @param tagger The tagger to delegate to. Must not be {@code null}. + * @throws IllegalArgumentException Thrown if {@code tagger} is {@code null}. + */ + public POSTaggerAnnotator(POSTagger tagger) { + if (tagger == null) { + throw new IllegalArgumentException("tagger must not be null"); + } + this.tagger = tagger; + } + + @Override + public Document annotate(Document document) { + if (document == null) { + throw new IllegalArgumentException("document must not be null"); + } + final List<Annotation<String>> tokens = document.get(Layers.TOKENS); + if (tokens.isEmpty()) { + throw new IllegalArgumentException("document lacks the required layer " + + Layers.TOKENS); + } + final String[] words = new String[tokens.size()]; + for (int i = 0; i < words.length; i++) { + words[i] = tokens.get(i).value(); + } + final String[] tags = tagger.tag(words); + final List<Annotation<String>> tagAnnotations = new ArrayList<>(tags.length); + for (int i = 0; i < tags.length; i++) { + tagAnnotations.add(new Annotation<>(tokens.get(i).span(), tags[i])); + } + return document.with(Layers.POS_TAGS, tagAnnotations); + } + + @Override + public Set<LayerKey<?>> requires() { + return Set.of(Layers.TOKENS); + } + + @Override + public Set<LayerKey<?>> provides() { + return Set.of(Layers.POS_TAGS); + } +} diff --git a/opennlp-api/src/main/java/opennlp/tools/document/SentenceDetectorAnnotator.java b/opennlp-api/src/main/java/opennlp/tools/document/SentenceDetectorAnnotator.java new file mode 100644 index 000000000..47e7c58c3 --- /dev/null +++ b/opennlp-api/src/main/java/opennlp/tools/document/SentenceDetectorAnnotator.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.document; + +import java.util.ArrayList; +import java.util.List; +import java.util.Set; + +import opennlp.tools.sentdetect.SentenceDetector; +import opennlp.tools.util.Span; + +/** + * Adapts a {@link SentenceDetector} to the document pipeline: provides + * {@link Layers#SENTENCES} from the document text. + * + * <p>The wrapped detector stays the primary API for single-task use; this adapter calls + * it like any other caller would.</p> + * + * @since 3.0.0 + */ +public class SentenceDetectorAnnotator implements DocumentAnnotator { + + private final SentenceDetector detector; + + /** + * Initializes the adapter. + * + * @param detector The sentence detector to delegate to. Must not be {@code null}. + * @throws IllegalArgumentException Thrown if {@code detector} is {@code null}. + */ + public SentenceDetectorAnnotator(SentenceDetector detector) { + if (detector == null) { + throw new IllegalArgumentException("detector must not be null"); + } + this.detector = detector; + } + + @Override + public Document annotate(Document document) { + if (document == null) { + throw new IllegalArgumentException("document must not be null"); + } + final CharSequence text = document.text(); + final List<Annotation<String>> sentences = new ArrayList<>(); + for (final Span span : detector.sentPosDetect(text)) { + sentences.add(new Annotation<>(span, span.getCoveredText(text).toString())); + } + return document.with(Layers.SENTENCES, sentences); + } + + @Override + public Set<LayerKey<?>> provides() { + return Set.of(Layers.SENTENCES); + } +} diff --git a/opennlp-api/src/main/java/opennlp/tools/document/TokenizerAnnotator.java b/opennlp-api/src/main/java/opennlp/tools/document/TokenizerAnnotator.java new file mode 100644 index 000000000..625410a18 --- /dev/null +++ b/opennlp-api/src/main/java/opennlp/tools/document/TokenizerAnnotator.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.document; + +import java.util.ArrayList; +import java.util.List; +import java.util.Set; + +import opennlp.tools.tokenize.Tokenizer; +import opennlp.tools.util.Span; + +/** + * Adapts a {@link Tokenizer} to the document pipeline: provides {@link Layers#TOKENS}. + * + * <p>When {@link Layers#SENTENCES} is present, each sentence is tokenized separately and + * the token spans are shifted back to document coordinates; otherwise the whole text is + * tokenized at once. Either way, every token span refers to the original document + * text.</p> + * + * @since 3.0.0 + */ +public class TokenizerAnnotator implements DocumentAnnotator { + + private final Tokenizer tokenizer; + + /** + * Initializes the adapter. + * + * @param tokenizer The tokenizer to delegate to. Must not be {@code null}. + * @throws IllegalArgumentException Thrown if {@code tokenizer} is {@code null}. + */ + public TokenizerAnnotator(Tokenizer tokenizer) { + if (tokenizer == null) { + throw new IllegalArgumentException("tokenizer must not be null"); + } + this.tokenizer = tokenizer; + } + + @Override + public Document annotate(Document document) { + if (document == null) { + throw new IllegalArgumentException("document must not be null"); + } + final String text = document.text().toString(); + final List<Annotation<String>> tokens = new ArrayList<>(); + final List<Annotation<String>> sentences = document.get(Layers.SENTENCES); + if (sentences.isEmpty()) { + addTokens(tokens, text, 0); + } else { + for (final Annotation<String> sentence : sentences) { + final Span span = sentence.span(); + addTokens(tokens, text.substring(span.getStart(), span.getEnd()), span.getStart()); + } + } + return document.with(Layers.TOKENS, tokens); + } + + private void addTokens(List<Annotation<String>> tokens, String text, int offset) { + for (final Span span : tokenizer.tokenizePos(text)) { + final Span shifted = new Span(span.getStart() + offset, span.getEnd() + offset); + tokens.add(new Annotation<>(shifted, span.getCoveredText(text).toString())); + } + } + + @Override + public Set<LayerKey<?>> provides() { + return Set.of(Layers.TOKENS); + } +} diff --git a/opennlp-api/src/test/java/opennlp/tools/document/DocumentAnalyzerTest.java b/opennlp-api/src/test/java/opennlp/tools/document/DocumentAnalyzerTest.java new file mode 100644 index 000000000..534fe115c --- /dev/null +++ b/opennlp-api/src/test/java/opennlp/tools/document/DocumentAnalyzerTest.java @@ -0,0 +1,194 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.document; + +import java.util.List; +import java.util.Set; + +import org.junit.jupiter.api.Test; + +import opennlp.tools.postag.POSTagger; +import opennlp.tools.sentdetect.SentenceDetector; +import opennlp.tools.tokenize.Tokenizer; +import opennlp.tools.util.Span; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +/** + * Tests the {@link DocumentAnalyzer} pipeline over the adapter annotators, using simple + * inline implementations of the task interfaces: whitespace tokenization, period sentence + * splitting, and a dictionary tagger. The point under test is the pipeline mechanics and + * span arithmetic, not model quality. + */ +public class DocumentAnalyzerTest { + + private static final SentenceDetector SPLITTER = new SentenceDetector() { + + @Override + public String[] sentDetect(CharSequence s) { + throw new UnsupportedOperationException(); + } + + @Override + public Span[] sentPosDetect(CharSequence s) { + // split after each period; keep it simple for the test + final String text = s.toString(); + int start = 0; + final java.util.List<Span> spans = new java.util.ArrayList<>(); + for (int i = 0; i < text.length(); i++) { + if (text.charAt(i) == '.') { + spans.add(new Span(start, i + 1)); + start = i + 2; + } + } + return spans.toArray(new Span[0]); + } + }; + + private static final Tokenizer WHITESPACE = new Tokenizer() { + + @Override + public String[] tokenize(String s) { + throw new UnsupportedOperationException(); + } + + @Override + public Span[] tokenizePos(String s) { + final java.util.List<Span> spans = new java.util.ArrayList<>(); + int start = -1; + for (int i = 0; i <= s.length(); i++) { + final boolean boundary = i == s.length() || s.charAt(i) == ' '; + if (boundary && start >= 0) { + spans.add(new Span(start, i)); + start = -1; + } else if (!boundary && start < 0) { + start = i; + } + } + return spans.toArray(new Span[0]); + } + }; + + private static final POSTagger TAGGER = new POSTagger() { + + @Override + public String[] tag(String[] sentence) { + final String[] tags = new String[sentence.length]; + for (int i = 0; i < sentence.length; i++) { + tags[i] = "barks.".contains(sentence[i]) ? "VBZ" : "X"; + } + return tags; + } + + @Override + public String[] tag(String[] sentence, Object[] additionalContext) { + return tag(sentence); + } + + @Override + public opennlp.tools.util.Sequence[] topKSequences(String[] sentence) { + throw new UnsupportedOperationException(); + } + + @Override + public opennlp.tools.util.Sequence[] topKSequences(String[] sentence, + Object[] additionalContext) { + throw new UnsupportedOperationException(); + } + }; + + @Test + void testPipelineProducesAlignedLayersInOriginalCoordinates() { + final Document document = DocumentAnalyzer.builder() + .add(new SentenceDetectorAnnotator(SPLITTER)) + .add(new TokenizerAnnotator(WHITESPACE)) + .add(new POSTaggerAnnotator(TAGGER)) + .build() + .analyze("the dog barks. she eats."); + + final List<Annotation<String>> sentences = document.get(Layers.SENTENCES); + assertEquals(2, sentences.size()); + assertEquals("she eats.", sentences.get(1).value()); + + final List<Annotation<String>> tokens = document.get(Layers.TOKENS); + assertEquals(5, tokens.size()); + // token of the second sentence, span in document coordinates + assertEquals("she", tokens.get(3).value()); + assertEquals(new Span(15, 18), tokens.get(3).span()); + + final List<Annotation<String>> tags = document.get(Layers.POS_TAGS); + assertEquals(5, tags.size()); + assertEquals("VBZ", tags.get(2).value()); + assertEquals(tokens.get(2).span(), tags.get(2).span()); + } + + @Test + void testTokenizerWorksWithoutSentences() { + final Document document = DocumentAnalyzer.builder() + .add(new TokenizerAnnotator(WHITESPACE)) + .build() + .analyze("the dog"); + assertEquals(2, document.get(Layers.TOKENS).size()); + } + + @Test + void testMisorderedPipelineFailsAtBuildTime() { + final DocumentAnalyzer.Builder builder = DocumentAnalyzer.builder() + .add(new POSTaggerAnnotator(TAGGER)); + assertThrows(IllegalArgumentException.class, builder::build); + } + + @Test + void testEmptyPipelineThrows() { + assertThrows(IllegalArgumentException.class, () -> DocumentAnalyzer.builder().build()); + } + + @Test + void testCustomLayerNeedsNoContainerChange() { + // the additive claim: a brand-new layer type works without touching the container + record Sentiment(String polarity, double score) { + } + final LayerKey<Sentiment> sentiment = LayerKey.of("sentiment", Sentiment.class); + final DocumentAnnotator annotator = new DocumentAnnotator() { + + @Override + public Document annotate(Document document) { + final Span all = new Span(0, document.text().length()); + return document.with(sentiment, + List.of(new Annotation<>(all, new Sentiment("positive", 0.9d)))); + } + + @Override + public Set<LayerKey<?>> provides() { + return Set.of(sentiment); + } + }; + final Document document = DocumentAnalyzer.builder().add(annotator).build() + .analyze("good dog"); + assertEquals("positive", document.get(sentiment).get(0).value().polarity()); + } + + @Test + void testAnnotatorAdaptersRejectNullDelegates() { + assertThrows(IllegalArgumentException.class, () -> new SentenceDetectorAnnotator(null)); + assertThrows(IllegalArgumentException.class, () -> new TokenizerAnnotator(null)); + assertThrows(IllegalArgumentException.class, () -> new POSTaggerAnnotator(null)); + assertThrows(IllegalArgumentException.class, () -> new NameFinderAnnotator(null)); + } +} diff --git a/opennlp-api/src/test/java/opennlp/tools/document/DocumentTest.java b/opennlp-api/src/test/java/opennlp/tools/document/DocumentTest.java new file mode 100644 index 000000000..5af4908bc --- /dev/null +++ b/opennlp-api/src/test/java/opennlp/tools/document/DocumentTest.java @@ -0,0 +1,125 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.document; + +import java.util.List; +import java.util.Set; + +import org.junit.jupiter.api.Test; + +import opennlp.tools.util.Span; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Tests the {@link Document} container: typed layer access, copy-on-add immutability, + * and the insertion-time validation that protects the layer invariants. + */ +public class DocumentTest { + + private static final LayerKey<String> WORDS = LayerKey.of("words", String.class); + private static final LayerKey<Integer> NUMBERS = LayerKey.of("numbers", Integer.class); + + @Test + void testEmptyDocument() { + final Document document = Document.of("the dog"); + assertEquals("the dog", document.text()); + assertTrue(document.layers().isEmpty()); + assertTrue(document.get(WORDS).isEmpty()); + } + + @Test + void testWithAddsATypedLayer() { + final Document document = Document.of("the dog") + .with(WORDS, List.of(new Annotation<>(new Span(0, 3), "the"), + new Annotation<>(new Span(4, 7), "dog"))); + assertEquals(Set.of(WORDS), document.layers()); + final List<Annotation<String>> words = document.get(WORDS); + assertEquals(2, words.size()); + assertEquals("dog", words.get(1).value()); + assertEquals(new Span(4, 7), words.get(1).span()); + } + + @Test + void testWithIsCopyOnAdd() { + final Document empty = Document.of("42"); + final Document grown = empty.with(NUMBERS, + List.of(new Annotation<>(new Span(0, 2), 42))); + assertTrue(empty.layers().isEmpty()); + assertEquals(Set.of(NUMBERS), grown.layers()); + // unchanged layers are shared, not copied + final Document both = grown.with(WORDS, List.of()); + assertSame(grown.get(NUMBERS), both.get(NUMBERS)); + } + + @Test + void testEqualKeysFromDifferentConstantsInteroperate() { + final Document document = Document.of("the") + .with(LayerKey.of("words", String.class), + List.of(new Annotation<>(new Span(0, 3), "the"))); + assertEquals(1, document.get(WORDS).size()); + assertNotEquals(WORDS, LayerKey.of("words", CharSequence.class)); + } + + @Test + void testDuplicateLayerThrows() { + final Document document = Document.of("the").with(WORDS, List.of()); + assertThrows(IllegalArgumentException.class, () -> document.with(WORDS, List.of())); + } + + @Test + void testSpanBeyondTextThrows() { + assertThrows(IllegalArgumentException.class, () -> Document.of("the") + .with(WORDS, List.of(new Annotation<>(new Span(0, 4), "the?")))); + } + + @Test + void testValueTypeIsCheckedOnInsertion() { + // a raw-typed caller cannot smuggle a mismatched value past the layer type + @SuppressWarnings({"unchecked", "rawtypes"}) + final LayerKey<Object> raw = (LayerKey) NUMBERS; + assertThrows(IllegalArgumentException.class, () -> Document.of("the") + .with(raw, List.of(new Annotation<>(new Span(0, 3), "not a number")))); + } + + @Test + void testNullArgumentsThrow() { + final Document document = Document.of("the"); + assertThrows(IllegalArgumentException.class, () -> Document.of(null)); + assertThrows(IllegalArgumentException.class, () -> document.get(null)); + assertThrows(IllegalArgumentException.class, () -> document.with(null, List.of())); + assertThrows(IllegalArgumentException.class, () -> document.with(WORDS, null)); + } + + @Test + void testAnnotationValidation() { + assertThrows(IllegalArgumentException.class, () -> new Annotation<>(null, "the")); + assertThrows(IllegalArgumentException.class, () -> new Annotation<>(new Span(0, 3), null)); + } + + @Test + void testLayerKeyValidation() { + assertThrows(IllegalArgumentException.class, () -> LayerKey.of(" ", String.class)); + assertThrows(IllegalArgumentException.class, () -> LayerKey.of(null, String.class)); + assertThrows(IllegalArgumentException.class, () -> LayerKey.of("words", null)); + } +} diff --git a/opennlp-api/src/test/java/opennlp/tools/document/NameFinderAnnotatorTest.java b/opennlp-api/src/test/java/opennlp/tools/document/NameFinderAnnotatorTest.java new file mode 100644 index 000000000..38478ac90 --- /dev/null +++ b/opennlp-api/src/test/java/opennlp/tools/document/NameFinderAnnotatorTest.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.document; + +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; + +import org.junit.jupiter.api.Test; + +import opennlp.tools.namefind.TokenNameFinder; +import opennlp.tools.util.Span; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +/** + * Tests that {@link NameFinderAnnotator} maps token-index mentions to character spans on + * the original text and clears the finder's adaptive data per document. + */ +public class NameFinderAnnotatorTest { + + @Test + void testTokenIndexSpansBecomeCharacterSpans() { + final AtomicInteger cleared = new AtomicInteger(); + final TokenNameFinder finder = new TokenNameFinder() { + + @Override + public Span[] find(String[] tokens) { + // "New York" as a two-token person-free location mention + return new Span[] {new Span(1, 3, "location")}; + } + + @Override + public void clearAdaptiveData() { + cleared.incrementAndGet(); + } + }; + + final Document document = Document.of("in New York today") + .with(Layers.TOKENS, List.of( + new Annotation<>(new Span(0, 2), "in"), + new Annotation<>(new Span(3, 6), "New"), + new Annotation<>(new Span(7, 11), "York"), + new Annotation<>(new Span(12, 17), "today"))); + + final Document annotated = new NameFinderAnnotator(finder).annotate(document); + final List<Annotation<String>> entities = annotated.get(Layers.ENTITIES); + assertEquals(1, entities.size()); + assertEquals(new Span(3, 11, "location"), entities.get(0).span()); + assertEquals("location", entities.get(0).value()); + assertEquals("New York", + entities.get(0).span().getCoveredText(annotated.text()).toString()); + assertEquals(1, cleared.get()); + } + + @Test + void testMissingTokenLayerThrows() { + final TokenNameFinder finder = new TokenNameFinder() { + + @Override + public Span[] find(String[] tokens) { + return new Span[0]; + } + + @Override + public void clearAdaptiveData() { + } + }; + assertThrows(IllegalArgumentException.class, + () -> new NameFinderAnnotator(finder).annotate(Document.of("no tokens"))); + } +}
