This is an automated email from the ASF dual-hosted git repository.
mawiesne pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/opennlp.git
The following commit(s) were added to refs/heads/main by this push:
new 64a8439a5 OPENNLP-587: Document the Entity Linker framework (#1169)
64a8439a5 is described below
commit 64a8439a5ef88f0df242d8dce6b80619ae536e97
Author: jerry317395616 <[email protected]>
AuthorDate: Thu Jul 16 14:46:30 2026 +0800
OPENNLP-587: Document the Entity Linker framework (#1169)
---
.../EntityLinkerManualExamplesTest.java | 177 +++++++++++++++++
opennlp-docs/src/docbkx/cli.xml | 8 +
opennlp-docs/src/docbkx/entitylinker.xml | 213 +++++++++++++++++++++
opennlp-docs/src/docbkx/opennlp.xml | 1 +
4 files changed, 399 insertions(+)
diff --git
a/opennlp-api/src/test/java/opennlp/tools/entitylinker/EntityLinkerManualExamplesTest.java
b/opennlp-api/src/test/java/opennlp/tools/entitylinker/EntityLinkerManualExamplesTest.java
new file mode 100644
index 000000000..40f97f54d
--- /dev/null
+++
b/opennlp-api/src/test/java/opennlp/tools/entitylinker/EntityLinkerManualExamplesTest.java
@@ -0,0 +1,177 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.entitylinker;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import org.junit.jupiter.api.Test;
+
+import opennlp.tools.namefind.TokenNameFinder;
+import opennlp.tools.sentdetect.SentenceDetector;
+import opennlp.tools.tokenize.Tokenizer;
+import opennlp.tools.tokenize.WhitespaceTokenizer;
+import opennlp.tools.util.Span;
+
+import static org.junit.jupiter.api.Assertions.assertAll;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+public class EntityLinkerManualExamplesTest {
+
+ private static final String DOCUMENT_TEXT = "Visit Paris\nThen New York";
+
+ @Test
+ void testNameTokenIndexesBecomeDocumentOffsets() {
+ EntityLinker<LinkedSpan<GazetteerLink>> linker = new
GazetteerLinker(Map.of(
+ "Paris", new GazetteerLink("Q90", "Paris"),
+ "New York", new GazetteerLink("Q60", "New York")));
+
+ List<? extends Span> results = linkEntities(DOCUMENT_TEXT, new
FixedSentenceDetector(),
+ WhitespaceTokenizer.INSTANCE, new FixedNameFinder(), linker);
+
+ assertEquals(2, results.size());
+ assertLinkedSpan((LinkedSpan<?>) results.get(0), 6, 11, 0, "Paris", "Q90");
+ assertLinkedSpan((LinkedSpan<?>) results.get(1), 17, 25, 1, "New York",
"Q60");
+ }
+
+ private static List<? extends Span> linkEntities(String documentText,
+ SentenceDetector sentenceDetector, Tokenizer tokenizer,
+ TokenNameFinder nameFinder, EntityLinker<? extends Span> linker) {
+
+ Span[] sentences = sentenceDetector.sentPosDetect(documentText);
+ Span[][] tokensBySentence = new Span[sentences.length][];
+ Span[][] namesBySentence = new Span[sentences.length][];
+
+ for (int i = 0; i < sentences.length; i++) {
+ Span sentence = sentences[i];
+ String sentenceText = sentence.getCoveredText(documentText).toString();
+ Span[] relativeTokenSpans = tokenizer.tokenizePos(sentenceText);
+ String[] tokenTexts = Span.spansToStrings(relativeTokenSpans,
sentenceText);
+
+ namesBySentence[i] = nameFinder.find(tokenTexts);
+ tokensBySentence[i] = new Span[relativeTokenSpans.length];
+ for (int j = 0; j < relativeTokenSpans.length; j++) {
+ // Tokenizer offsets are sentence-relative; EntityLinker expects
document offsets.
+ tokensBySentence[i][j] = new Span(relativeTokenSpans[j],
sentence.getStart());
+ }
+ }
+
+ return linker.find(documentText, sentences, tokensBySentence,
namesBySentence);
+ }
+
+ private static void assertLinkedSpan(LinkedSpan<?> span, int start, int end,
+ int sentenceId, String searchTerm, String itemId) {
+ assertAll(
+ () -> assertEquals(start, span.getStart()),
+ () -> assertEquals(end, span.getEnd()),
+ () -> assertEquals("location", span.getType()),
+ () -> assertEquals(sentenceId, span.getSentenceid()),
+ () -> assertEquals(searchTerm, span.getSearchTerm()),
+ () -> assertEquals(itemId,
span.getLinkedEntries().get(0).getItemID()));
+ }
+
+ private static final class FixedSentenceDetector implements SentenceDetector
{
+
+ @Override
+ public String[] sentDetect(CharSequence text) {
+ return Span.spansToStrings(sentPosDetect(text), text);
+ }
+
+ @Override
+ public Span[] sentPosDetect(CharSequence text) {
+ return new Span[] {new Span(0, 11), new Span(12, 25)};
+ }
+ }
+
+ private static final class FixedNameFinder implements TokenNameFinder {
+
+ @Override
+ public Span[] find(String[] tokens) {
+ if (tokens.length == 2) {
+ return new Span[] {new Span(1, 2, "location")};
+ }
+ return new Span[] {new Span(1, 3, "location")};
+ }
+
+ @Override
+ public void clearAdaptiveData() {
+ // The fixed test implementation does not retain adaptive data.
+ }
+ }
+
+ private static final class GazetteerLink extends BaseLink {
+
+ private GazetteerLink(String id, String name) {
+ super(null, id, name, "location");
+ }
+ }
+
+ private static final class GazetteerLinker
+ implements EntityLinker<LinkedSpan<GazetteerLink>> {
+
+ private final Map<String, GazetteerLink> entries;
+
+ private GazetteerLinker(Map<String, GazetteerLink> entries) {
+ this.entries = entries;
+ }
+
+ @Override
+ public void init(EntityLinkerProperties properties) {
+ // The test linker receives its entries in memory and needs no external
initialization.
+ }
+
+ @Override
+ public List<LinkedSpan<GazetteerLink>> find(String documentText,
+ Span[] sentences, Span[][] tokensBySentence, Span[][] namesBySentence)
{
+ List<LinkedSpan<GazetteerLink>> results = new ArrayList<>();
+ for (int i = 0; i < sentences.length; i++) {
+ results.addAll(find(documentText, sentences, tokensBySentence,
+ namesBySentence, i));
+ }
+ return results;
+ }
+
+ @Override
+ public List<LinkedSpan<GazetteerLink>> find(String documentText,
+ Span[] sentences, Span[][] tokensBySentence, Span[][] namesBySentence,
+ int sentenceIndex) {
+ List<LinkedSpan<GazetteerLink>> results = new ArrayList<>();
+ Span[] tokens = tokensBySentence[sentenceIndex];
+
+ for (Span name : namesBySentence[sentenceIndex]) {
+ // Name spans contain token indexes, so map their boundaries through
the token spans.
+ int start = tokens[name.getStart()].getStart();
+ int end = tokens[name.getEnd() - 1].getEnd();
+ String searchTerm = documentText.substring(start, end);
+ GazetteerLink match = entries.get(searchTerm);
+
+ if (match != null) {
+ ArrayList<GazetteerLink> matches = new ArrayList<>();
+ matches.add(match);
+ LinkedSpan<GazetteerLink> linkedSpan =
+ new LinkedSpan<>(matches, start, end, name.getType());
+ linkedSpan.setSentenceid(sentenceIndex);
+ linkedSpan.setSearchTerm(searchTerm);
+ results.add(linkedSpan);
+ }
+ }
+ return results;
+ }
+ }
+}
diff --git a/opennlp-docs/src/docbkx/cli.xml b/opennlp-docs/src/docbkx/cli.xml
index 23d44a070..0c29307cf 100644
--- a/opennlp-docs/src/docbkx/cli.xml
+++ b/opennlp-docs/src/docbkx/cli.xml
@@ -4381,6 +4381,14 @@ Arguments description:
<para>Links an entity to an external data set</para>
+ <para>
+ The <emphasis>model</emphasis> argument is an Entity Linker properties file.
The command loads
+ the implementation named by <literal>linker.location</literal> and reads
tokenized,
+ named-entity-annotated sentences from standard input. See the
+ <link linkend="tools.entitylinker.commandline">Entity Linker chapter</link>
for the input format
+ and configuration details.
+ </para>
+
<screen>
<![CDATA[Usage: opennlp EntityLinker model < sentences]]>
</screen>
diff --git a/opennlp-docs/src/docbkx/entitylinker.xml
b/opennlp-docs/src/docbkx/entitylinker.xml
new file mode 100644
index 000000000..628d9a7d5
--- /dev/null
+++ b/opennlp-docs/src/docbkx/entitylinker.xml
@@ -0,0 +1,213 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE chapter PUBLIC "-//OASIS//DTD DocBook XML 5.0//EN"
+"http://docbook.org/xml/5.0/dtd/docbook.dtd";[
+]>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+<chapter xml:id="tools.entitylinker"
xmlns:xlink="http://www.w3.org/1999/xlink";>
+
+ <title>Entity Linker</title>
+
+ <section xml:id="tools.entitylinker.overview">
+ <title>Overview</title>
+ <para>
+ The Entity Linker API lets an application associate named
entities with records in an
+ external data source. For example, a location found by a Name
Finder can be matched to a
+ gazetteer entry, or a person can be matched to a directory
record. OpenNLP provides the
+ linking interfaces and data types; applications supply the
implementation and data source.
+ </para>
+ <para>
+ An <classname>EntityLinker</classname> receives the document
text, sentence and token spans,
+ and the name spans produced for each sentence. It returns
application-defined spans that hold
+ the links. <classname>LinkedSpan</classname> and
<classname>BaseLink</classname> are support
+ classes for this result; they are not linker implementations.
+ </para>
+ </section>
+
+ <section xml:id="tools.entitylinker.inputs">
+ <title>Preparing Input Spans</title>
+ <para>
+ The four arguments passed to <methodname>find</methodname> use
two coordinate systems. Sentence
+ and token spans are character offsets in the complete document.
Name spans are token indexes
+ within their sentence, as returned by
<classname>NameFinderME</classname>. The outer indexes of
+ <varname>tokensBySentence</varname> and
<varname>namesBySentence</varname> must correspond to the
+ same entry in <varname>sentences</varname>.
+ </para>
+ <para>
+ The following method prepares those arrays. The sentence
detector, tokenizer, and name finder
+ are accepted as interfaces so the same code works with their
standard OpenNLP implementations.
+ <programlisting language="java">
+<![CDATA[static List<? extends Span> linkEntities(String documentText,
+ SentenceDetector sentenceDetector, Tokenizer tokenizer,
+ TokenNameFinder nameFinder, EntityLinker<? extends Span> linker) {
+
+ Span[] sentences = sentenceDetector.sentPosDetect(documentText);
+ Span[][] tokensBySentence = new Span[sentences.length][];
+ Span[][] namesBySentence = new Span[sentences.length][];
+
+ for (int i = 0; i < sentences.length; i++) {
+ Span sentence = sentences[i];
+ String sentenceText = sentence.getCoveredText(documentText).toString();
+ Span[] relativeTokenSpans = tokenizer.tokenizePos(sentenceText);
+ String[] tokenTexts = Span.spansToStrings(relativeTokenSpans,
sentenceText);
+
+ namesBySentence[i] = nameFinder.find(tokenTexts);
+ tokensBySentence[i] = new Span[relativeTokenSpans.length];
+ for (int j = 0; j < relativeTokenSpans.length; j++) {
+ // Tokenizer offsets are sentence-relative; EntityLinker expects
document offsets.
+ tokensBySentence[i][j] = new Span(relativeTokenSpans[j],
sentence.getStart());
+ }
+ }
+
+ return linker.find(documentText, sentences, tokensBySentence,
namesBySentence);
+}]]>
+ </programlisting>
+ </para>
+ </section>
+
+ <section xml:id="tools.entitylinker.implementing">
+ <title>Implementing an Entity Linker</title>
+ <para>
+ Implement <classname>EntityLinker<T></classname>, where
<classname>T</classname> extends
+ <classname>Span</classname>. Load reusable resources in
<methodname>init</methodname>. Implement
+ the document overload of <methodname>find</methodname> by
visiting each sentence, and implement
+ the sentence overload with the actual lookup logic.
+ </para>
+ <para>
+ This example uses <classname>LinkedSpan</classname> for its
results and a small concrete
+ subclass of <classname>BaseLink</classname> for gazetteer
records:
+ <programlisting language="java">
+<![CDATA[public final class GazetteerLink extends BaseLink {
+
+ public GazetteerLink(String id, String name) {
+ super(null, id, name, "location");
+ }
+}
+
+public final class GazetteerLinker
+ implements EntityLinker<LinkedSpan<GazetteerLink>> {
+
+ private Map<String, GazetteerLink> entries;
+
+ @Override
+ public void init(EntityLinkerProperties properties) throws IOException {
+ String path = properties.getProperty("gazetteer.path", "gazetteer.tsv");
+ entries = loadEntries(Path.of(path));
+ }
+
+ @Override
+ public List<LinkedSpan<GazetteerLink>> find(String documentText,
+ Span[] sentences, Span[][] tokensBySentence, Span[][] namesBySentence) {
+ List<LinkedSpan<GazetteerLink>> results = new ArrayList<>();
+ for (int i = 0; i < sentences.length; i++) {
+ results.addAll(find(documentText, sentences, tokensBySentence,
+ namesBySentence, i));
+ }
+ return results;
+ }
+
+ @Override
+ public List<LinkedSpan<GazetteerLink>> find(String documentText,
+ Span[] sentences, Span[][] tokensBySentence, Span[][] namesBySentence,
+ int sentenceIndex) {
+ List<LinkedSpan<GazetteerLink>> results = new ArrayList<>();
+ Span[] tokens = tokensBySentence[sentenceIndex];
+
+ for (Span name : namesBySentence[sentenceIndex]) {
+ // Name spans contain token indexes, so map their boundaries through the
token spans.
+ int start = tokens[name.getStart()].getStart();
+ int end = tokens[name.getEnd() - 1].getEnd();
+ String searchTerm = documentText.substring(start, end);
+ GazetteerLink match = entries.get(searchTerm);
+
+ if (match != null) {
+ ArrayList<GazetteerLink> matches = new ArrayList<>();
+ matches.add(match);
+ LinkedSpan<GazetteerLink> linkedSpan =
+ new LinkedSpan<>(matches, start, end, name.getType());
+ linkedSpan.setSentenceid(sentenceIndex);
+ linkedSpan.setSearchTerm(searchTerm);
+ results.add(linkedSpan);
+ }
+ }
+ return results;
+ }
+
+ private static Map<String, GazetteerLink> loadEntries(Path path)
+ throws IOException {
+ // Parse the application's data source here.
+ return Map.of();
+ }
+}]]>
+ </programlisting>
+ </para>
+ <para>
+ A production implementation should validate array lengths and
name boundaries before indexing
+ into <varname>tokens</varname>, normalize lookup keys
consistently, and define how multiple
+ candidate records are scored and ordered.
+ </para>
+ </section>
+
+ <section xml:id="tools.entitylinker.configuration">
+ <title>Loading an Implementation</title>
+ <para>
+ <classname>EntityLinkerFactory</classname> loads an
implementation through OpenNLP's extension
+ loader and then calls its <methodname>init</methodname> method.
With the overload that accepts an
+ entity type, the key is
<literal>linker.<entityType></literal>:
+ <screen>
+<![CDATA[linker.location=com.example.GazetteerLinker
+gazetteer.path=/data/locations.tsv]]>
+ </screen>
+ </para>
+ <para>
+ Load the properties and request the configured linker as
follows:
+ <programlisting language="java">
+<![CDATA[EntityLinkerProperties properties =
+ new EntityLinkerProperties(new File("entity-linker.properties"));
+EntityLinker<?> linker = EntityLinkerFactory.getLinker("location",
properties);]]>
+ </programlisting>
+ The overload without an entity type reads the key
<literal>linker</literal> instead.
+ </para>
+ </section>
+
+ <section xml:id="tools.entitylinker.commandline">
+ <title>Command Line Tool</title>
+ <para>
+ The <command>EntityLinker</command> command loads the
<literal>linker.location</literal>
+ implementation from a properties file. It reads
blank-line-separated documents in the native
+ Name Finder sample format from standard input and writes each
linked span to standard output.
+ </para>
+ <screen>
+<![CDATA[opennlp EntityLinker entity-linker.properties < names.txt]]>
+ </screen>
+ <para>
+ Each non-empty input line is a tokenized sentence whose names
use
+ <literal><START:type></literal> and
<literal><END></literal> markers. For example:
+ <screen>
+<![CDATA[The office is in <START:location> New York <END> .
+
+The second document starts after the blank line .
+
+]]>
+ </screen>
+ </para>
+ </section>
+
+</chapter>
diff --git a/opennlp-docs/src/docbkx/opennlp.xml
b/opennlp-docs/src/docbkx/opennlp.xml
index 5d0166c5e..263c5f96c 100644
--- a/opennlp-docs/src/docbkx/opennlp.xml
+++ b/opennlp-docs/src/docbkx/opennlp.xml
@@ -111,6 +111,7 @@ under the License.
<xi:include xmlns:xi="http://www.w3.org/2001/XInclude";
href="./spellcheck.xml" />
<xi:include xmlns:xi="http://www.w3.org/2001/XInclude";
href="./chunker.xml" />
<xi:include xmlns:xi="http://www.w3.org/2001/XInclude";
href="./parser.xml" />
+ <xi:include xmlns:xi="http://www.w3.org/2001/XInclude";
href="./entitylinker.xml" />
<xi:include xmlns:xi="http://www.w3.org/2001/XInclude";
href="./coref.xml" />
<xi:include xmlns:xi="http://www.w3.org/2001/XInclude";
href="./model-loading.xml" />
<xi:include xmlns:xi="http://www.w3.org/2001/XInclude";
href="./extension.xml" />
