This is an automated email from the ASF dual-hosted git repository. krickert pushed a commit to branch wordnet-expansion in repository https://gitbox.apache.org/repos/asf/opennlp.git
commit 5688deba83bf3ae1f9791e6ca9dac0e2c87a54dd Author: Kristian Rickert <[email protected]> AuthorDate: Thu Jul 16 06:23:11 2026 -0400 OPENNLP-1887: Fold through the shared lemma fold, use the unknown-lemma constant, full helper javadoc --- .../main/java/opennlp/wordnet/LexicalExpander.java | 65 ++++++++++++++++------ .../java/opennlp/wordnet/ExpansionAssertions.java | 42 ++++++++++++++ .../wordnet/LexicalExpanderLexiconTest.java | 28 +++++++--- .../java/opennlp/wordnet/LexicalExpanderTest.java | 43 +++++++++----- 4 files changed, 139 insertions(+), 39 deletions(-) diff --git a/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/LexicalExpander.java b/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/LexicalExpander.java index 1e6c86442..0aae27c3e 100644 --- a/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/LexicalExpander.java +++ b/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/LexicalExpander.java @@ -22,7 +22,6 @@ import java.util.Comparator; import java.util.HashMap; import java.util.HashSet; import java.util.List; -import java.util.Locale; import java.util.Map; import java.util.Set; @@ -92,7 +91,11 @@ public final class LexicalExpander { private final double senseDecay; private final double depthDecay; - /** Creates an expander from a validated {@link Builder}. */ + /** + * Creates an expander from a builder whose fields have already been validated. + * + * @param builder The configured builder. Must not be {@code null}. + */ private LexicalExpander(Builder builder) { this.lexicon = builder.lexicon; this.lemmatizer = builder.lemmatizer; @@ -144,14 +147,22 @@ public final class LexicalExpander { return collect(term, List.of(WordNetPOS.values())); } - /** Expands the term across the given parts of speech and returns the ranked, capped result. */ + /** + * Expands the term across the given parts of speech and returns the ranked, capped result. + * + * @param term The term to expand. Must not be {@code null} or blank. + * @param poses The parts of speech to expand as. Must not be {@code null}. + * @return The expansions, deduplicated, ordered by descending weight, and capped at the + * configured maximum; empty when the term is not in the lexicon. + * @throws IllegalArgumentException Thrown if {@code term} is {@code null} or blank. + */ private List<Expansion> collect(String term, List<WordNetPOS> poses) { if (term == null || term.isBlank()) { throw new IllegalArgumentException("The term must not be null or blank."); } final Map<String, Expansion> best = new HashMap<>(); final Set<String> excluded = new HashSet<>(); - excluded.add(fold(term)); + excluded.add(LemmaFolding.fold(term)); for (final WordNetPOS pos : poses) { final String subject = resolveSubject(term, pos); @@ -176,7 +187,12 @@ public final class LexicalExpander { /** * Resolves the form actually expanded: the term when the lexicon knows it, otherwise its lemma - * when a lemmatizer is configured and produces a known lemma, otherwise {@code null}. + * when a lemmatizer is configured and produces a known lemma. + * + * @param term The input term. Must not be {@code null}. + * @param pos The part of speech to look the term up as. Must not be {@code null}. + * @return The known form to expand, or {@code null} when neither the term nor its lemma is in + * the lexicon. */ private String resolveSubject(String term, WordNetPOS pos) { if (lexicon.contains(term, pos)) { @@ -191,14 +207,23 @@ public final class LexicalExpander { return null; } final String lemma = lemmas[0]; - // "O" is the Lemmatizer contract's unknown marker. - if ("O".equals(lemma) || !lexicon.contains(lemma, pos)) { + // Lemmatizers report an unresolvable token with the contract's unknown marker. + if (MorphyLemmatizer.UNKNOWN_LEMMA.equals(lemma) || !lexicon.contains(lemma, pos)) { return null; } return lemma; } - /** Offers the synonyms, hypernym ancestors, and optional hyponyms of one sense into {@code best}. */ + /** + * Offers the synonyms, hypernym ancestors, and optional hyponyms of one sense into the running + * best-expansion map. + * + * @param sense The sense to expand. Must not be {@code null}. + * @param rank The zero-based salience rank of the sense. + * @param senseWeight The weight of the sense itself; each relation step decays from it. + * @param best The best expansion seen so far per folded term; updated in place. + * @param excluded The folded terms that are never reported, such as the input term. + */ private void expandSense(Synset sense, int rank, double senseWeight, Map<String, Expansion> best, Set<String> excluded) { for (final String lemma : sense.lemmas()) { @@ -247,7 +272,12 @@ public final class LexicalExpander { } } - /** Returns the synset ids of both the direct and the instance hypernyms of the synset. */ + /** + * Collects the synset ids of both the direct and the instance hypernyms of a synset. + * + * @param synset The synset whose hypernyms are collected. Must not be {@code null}. + * @return The hypernym synset ids in source order, direct relations first. + */ private static List<String> hypernymsOf(Synset synset) { final List<String> direct = synset.related(WordNetRelation.HYPERNYM); final List<String> instance = synset.related(WordNetRelation.INSTANCE_HYPERNYM); @@ -260,10 +290,18 @@ public final class LexicalExpander { return all; } - /** Records the candidate under its folded term when not excluded and it beats any current best. */ + /** + * Records the candidate under its folded term when it is not excluded and it beats the current + * best weight for that term. Folding through {@link LemmaFolding#fold(String)} keeps the + * exclusion and deduplication keys aligned with the lexicon's own lemma keys. + * + * @param best The best expansion seen so far per folded term; updated in place. + * @param excluded The folded terms that are never reported. + * @param candidate The expansion to offer. Must not be {@code null}. + */ private static void offer(Map<String, Expansion> best, Set<String> excluded, Expansion candidate) { - final String key = fold(candidate.term()); + final String key = LemmaFolding.fold(candidate.term()); if (excluded.contains(key)) { return; } @@ -273,11 +311,6 @@ public final class LexicalExpander { } } - /** Folds a term to its case-insensitive comparison key. */ - private static String fold(String term) { - return term.toLowerCase(Locale.ROOT); - } - /** Configures and creates a {@link LexicalExpander}. */ public static final class Builder { diff --git a/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/ExpansionAssertions.java b/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/ExpansionAssertions.java new file mode 100644 index 000000000..c67d7c425 --- /dev/null +++ b/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/ExpansionAssertions.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.wordnet; + +import java.util.List; + +import opennlp.wordnet.LexicalExpander.Expansion; + +/** + * Shared lookup helpers for tests that assert on {@link LexicalExpander} output. + */ +final class ExpansionAssertions { + + /** Not instantiable. */ + private ExpansionAssertions() { + } + + /** + * Finds the first expansion of a term in an expansion list. + * + * @param expansions The expansions to search. Must not be {@code null}. + * @param term The exact term to find. Must not be {@code null}. + * @return The first expansion whose term equals {@code term}, or {@code null} when absent. + */ + static Expansion find(List<Expansion> expansions, String term) { + return expansions.stream().filter(e -> e.term().equals(term)).findFirst().orElse(null); + } +} diff --git a/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/LexicalExpanderLexiconTest.java b/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/LexicalExpanderLexiconTest.java index 44506a1d5..c52b01da7 100644 --- a/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/LexicalExpanderLexiconTest.java +++ b/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/LexicalExpanderLexiconTest.java @@ -25,8 +25,9 @@ import opennlp.tools.wordnet.WordNetPOS; import opennlp.wordnet.LexicalExpander.Expansion; import opennlp.wordnet.LexicalExpander.Kind; +import static opennlp.wordnet.ExpansionAssertions.find; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.assertNotNull; /** * End-to-end expansion over the miniature lexicon fixtures: the WN-LMF and WNDB readers each @@ -35,10 +36,6 @@ import static org.junit.jupiter.api.Assertions.assertTrue; */ class LexicalExpanderLexiconTest { - private static Expansion find(List<Expansion> expansions, String term) { - return expansions.stream().filter(e -> e.term().equals(term)).findFirst().orElse(null); - } - @Test void testExpansionOverTheWnLmfLexicon() { final LexicalExpander expander = @@ -59,11 +56,24 @@ class LexicalExpanderLexiconTest { LexicalExpander.builder(WndbReaderTest.fixture()).build(); final List<Expansion> expansions = expander.expand("dog", WordNetPOS.NOUN); - assertTrue(find(expansions, "domestic dog") != null, "got " + expansions); + assertNotNull(find(expansions, "domestic dog"), "got " + expansions); final Expansion canid = find(expansions, "canid"); assertEquals(Kind.HYPERNYM, canid.kind()); } + @Test + void testUnderscoreQueryFoldsToTheMultiwordLexiconEntry() { + // The WNDB index stores the entry as "domestic_dog"; the reader folds it to "domestic dog" + // at load time. The expander must fold the underscore query the same way, so the entry's + // own synset expands and the query never surfaces as its own synonym. + final List<Expansion> expansions = LexicalExpander.builder(WndbReaderTest.fixture()) + .build().expand("domestic_dog", WordNetPOS.NOUN); + + assertEquals(List.of( + new Expansion("dog", Kind.SYNONYM, 0, 0, 1.0), + new Expansion("canid", Kind.HYPERNYM, 1, 0, 0.5)), expansions); + } + @Test void testReadersAgreeOnExpansions() { final List<Expansion> lmf = LexicalExpander.builder(WnLmfReaderTest.fixture()) @@ -74,7 +84,7 @@ class LexicalExpanderLexiconTest { assertEquals( lmf.stream().map(e -> e.term() + "|" + e.kind() + "|" + e.weight()).toList(), wndb.stream().map(e -> e.term() + "|" + e.kind() + "|" + e.weight()).toList()); - assertTrue(find(lmf, "rodent") != null, "got " + lmf); + assertNotNull(find(lmf, "rodent"), "got " + lmf); } @Test @@ -87,10 +97,10 @@ class LexicalExpanderLexiconTest { // A regular inflection resolves by rule, an irregular one by the exception list. final List<Expansion> dogs = expander.expand("dogs", WordNetPOS.NOUN); assertEquals(Kind.SYNONYM, find(dogs, "dog").kind()); - assertTrue(find(dogs, "canid") != null, "got " + dogs); + assertNotNull(find(dogs, "canid"), "got " + dogs); final List<Expansion> mice = expander.expand("mice", WordNetPOS.NOUN); assertEquals(Kind.SYNONYM, find(mice, "mouse").kind()); - assertTrue(find(mice, "rodent") != null, "got " + mice); + assertNotNull(find(mice, "rodent"), "got " + mice); } } diff --git a/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/LexicalExpanderTest.java b/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/LexicalExpanderTest.java index 9d30490c1..ee80dca47 100644 --- a/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/LexicalExpanderTest.java +++ b/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/LexicalExpanderTest.java @@ -18,7 +18,6 @@ package opennlp.wordnet; import java.util.HashMap; import java.util.List; -import java.util.Locale; import java.util.Map; import java.util.Optional; @@ -31,7 +30,10 @@ import opennlp.tools.wordnet.WordNetRelation; import opennlp.wordnet.LexicalExpander.Expansion; import opennlp.wordnet.LexicalExpander.Kind; +import static opennlp.wordnet.ExpansionAssertions.find; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -44,7 +46,9 @@ class LexicalExpanderTest { // dog: sense 1 = {dog, domestic dog} -> canid -> carnivore, with hyponym puppy; // sense 2 = {dog, frank, hot dog} -> sausage. The verb sense = {dog, chase}. + // hot dog: the standalone multiword sense {hot dog, red hot}. // alpha <-> beta form a malformed hypernym cycle. + // Lookups fold through LemmaFolding, exactly as the readers fold their keys at load time. private static LexicalKnowledgeBase lexicon() { final Map<String, Synset> synsets = new HashMap<>(); final Map<String, List<Synset>> senses = new HashMap<>(); @@ -63,17 +67,20 @@ class LexicalExpanderTest { Map.of()); final Synset v1 = new Synset("v1", WordNetPOS.VERB, List.of("dog", "chase"), "follow", Map.of()); + final Synset m1 = new Synset("m1", WordNetPOS.NOUN, List.of("hot dog", "red hot"), + "grilled sausage", Map.of()); final Synset c1 = new Synset("c1", WordNetPOS.NOUN, List.of("alpha"), "cycle start", Map.of(WordNetRelation.HYPERNYM, List.of("c2"))); final Synset c2 = new Synset("c2", WordNetPOS.NOUN, List.of("beta"), "cycle end", Map.of(WordNetRelation.HYPERNYM, List.of("c1"))); - for (final Synset synset : List.of(n1, n2, n3, n4, n5, n6, v1, c1, c2)) { + for (final Synset synset : List.of(n1, n2, n3, n4, n5, n6, v1, m1, c1, c2)) { synsets.put(synset.id(), synset); } senses.put("dog|NOUN", List.of(n1, n5)); senses.put("dog|VERB", List.of(v1)); senses.put("domestic dog|NOUN", List.of(n1)); + senses.put("hot dog|NOUN", List.of(m1)); senses.put("alpha|NOUN", List.of(c1)); return new LexicalKnowledgeBase() { @@ -82,7 +89,7 @@ class LexicalExpanderTest { if (lemma == null || pos == null) { throw new IllegalArgumentException("null"); } - return senses.getOrDefault(lemma.toLowerCase(Locale.ROOT) + "|" + pos, List.of()); + return senses.getOrDefault(LemmaFolding.fold(lemma) + "|" + pos, List.of()); } @Override @@ -92,10 +99,6 @@ class LexicalExpanderTest { }; } - private static Expansion find(List<Expansion> expansions, String term) { - return expansions.stream().filter(e -> e.term().equals(term)).findFirst().orElse(null); - } - @Test void testSynonymsAndHypernymsWithDefaultConfiguration() { final List<Expansion> expansions = @@ -117,8 +120,19 @@ class LexicalExpanderTest { assertEquals(0.5, frank.weight()); // Depth 1 by default: the grandparent stays out, and so do hyponyms. - assertEquals(null, find(expansions, "carnivore")); - assertEquals(null, find(expansions, "puppy")); + assertNull(find(expansions, "carnivore")); + assertNull(find(expansions, "puppy")); + } + + @Test + void testUnderscoreInputReachesTheSpaceFoldedLexiconEntry() { + // The lexicon keys "hot_dog" under its folded form "hot dog". The expander must fold the + // input the same way, so "hot dog" is excluded as the input itself and cannot displace the + // other member "red hot" from the capped result. + final List<Expansion> expansions = LexicalExpander.builder(lexicon()) + .maxExpansions(1).build().expand("hot_dog", WordNetPOS.NOUN); + + assertEquals(List.of(new Expansion("red hot", Kind.SYNONYM, 0, 0, 1.0)), expansions); } @Test @@ -154,8 +168,8 @@ class LexicalExpanderTest { final List<Expansion> expansions = LexicalExpander.builder(lexicon()) .maxSenses(1).build().expand("dog", WordNetPOS.NOUN); - assertEquals(null, find(expansions, "frank")); - assertTrue(find(expansions, "domestic dog") != null); + assertNull(find(expansions, "frank")); + assertNotNull(find(expansions, "domestic dog")); } @Test @@ -163,8 +177,8 @@ class LexicalExpanderTest { final List<Expansion> expansions = LexicalExpander.builder(lexicon()).build().expand("dog"); - assertTrue(find(expansions, "chase") != null); - assertTrue(find(expansions, "domestic dog") != null); + assertNotNull(find(expansions, "chase")); + assertNotNull(find(expansions, "domestic dog")); } @Test @@ -245,7 +259,7 @@ class LexicalExpanderTest { final Expansion dog = find(expansions, "dog"); assertEquals(Kind.SYNONYM, dog.kind()); assertEquals(1.0, dog.weight()); - assertTrue(find(expansions, "domestic dog") != null); + assertNotNull(find(expansions, "domestic dog")); assertEquals(List.of(), expander.expand("cats", WordNetPOS.NOUN)); } @@ -261,6 +275,7 @@ class LexicalExpanderTest { assertThrows(IllegalArgumentException.class, () -> builder.senseDecay(0)); assertThrows(IllegalArgumentException.class, () -> builder.senseDecay(1.5)); assertThrows(IllegalArgumentException.class, () -> builder.depthDecay(0)); + assertThrows(IllegalArgumentException.class, () -> builder.depthDecay(1.5)); final LexicalExpander expander = builder.build(); assertThrows(IllegalArgumentException.class, () -> expander.expand(null));
