This is an automated email from the ASF dual-hosted git repository. krickert pushed a commit to branch wordnet-expansion in repository https://gitbox.apache.org/repos/asf/opennlp.git
commit 9fcbac7f0639ca2a36bbb4a563fb68e697860535 Author: Kristian Rickert <[email protected]> AuthorDate: Fri Jul 10 15:26:56 2026 -0400 Add LexicalExpander: weighted synonym and hypernym expansion over the lexical knowledge base Expands a term into the synonyms sharing its synsets, the lemmas of its hypernym ancestors up to a configured depth (following both the direct and the instance relation), and optionally its direct hyponyms. Each expansion carries a deterministic heuristic weight: sense rank and every relation step multiply configurable decays, so consumers can discount looser expansions instead of treating them as the original term. Results exclude the input, deduplicate case-insensitively keeping the highest weight, and order stably by weight, kind, and term. Inflected input works through an optional Lemmatizer invoked with the WordNetPos name as the tag, which the Morphy lemmatizer understands: dogs expands through dog, mice through the exception list to mouse. Hypernym walks are visited-checked so malformed cyclic data terminates. Tests cover the graph behavior on a hand-built lexicon (sense ranking, decay arithmetic, dedup, cycle termination, validation) and run the whole stack over the miniature WN-LMF and WNDB fixtures, asserting both readers produce identical expansions. --- .../main/java/opennlp/wordnet/LexicalExpander.java | 412 +++++++++++++++++++++ .../wordnet/LexicalExpanderLexiconTest.java | 96 +++++ .../java/opennlp/wordnet/LexicalExpanderTest.java | 270 ++++++++++++++ 3 files changed, 778 insertions(+) diff --git a/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/LexicalExpander.java b/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/LexicalExpander.java new file mode 100644 index 000000000..d230d0668 --- /dev/null +++ b/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/LexicalExpander.java @@ -0,0 +1,412 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.wordnet; + +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; + +import opennlp.tools.commons.ThreadSafe; +import opennlp.tools.lemmatizer.Lemmatizer; +import opennlp.tools.wordnet.LexicalKnowledgeBase; +import opennlp.tools.wordnet.Synset; +import opennlp.tools.wordnet.WordNetPOS; +import opennlp.tools.wordnet.WordNetRelation; + +/** + * Expands a term into related terms drawn from a {@link LexicalKnowledgeBase}: the synonyms + * sharing its synsets, the lemmas of its hypernym ancestors up to a configured depth, and + * optionally the lemmas of its direct hyponyms. + * + * <p>Lexical expansion trades precision for recall: a consumer matching on {@code dog} can also + * match {@code domestic dog} (synonym) and {@code canid} (hypernym). Each {@link Expansion} + * carries a weight so the consumer can discount looser expansions instead of treating them as + * the original term. Weights are a deterministic heuristic, not probabilities: the first sense + * of a term starts at {@code 1.0}, each later sense is multiplied by the sense decay, and every + * hypernym or hyponym step multiplies by the depth decay. Both decays are configurable; the + * defaults halve the weight per step so distant ancestors fade quickly.</p> + * + * <p>When the term itself is not in the lexicon and a {@link Lemmatizer} is configured, the term + * is lemmatized and the lemma expanded instead, so inflected input ({@code dogs}, + * {@code running}) still expands and the lemma itself surfaces as a synonym expansion. The + * lemmatizer is invoked with the {@link WordNetPOS} name as the tag, which the Morphy + * lemmatizer of this module understands.</p> + * + * <p>Hypernym walks follow both the direct and the instance relation (an instance like a city + * name climbs to its class), track visited synsets so malformed cyclic data cannot loop, and + * never report the term itself. Results are deduplicated case-insensitively, keeping the highest + * weight, and ordered by weight descending, then kind, then term, so output is stable across + * runs.</p> + * + * <p>Instances are immutable and safe for concurrent use when the configured lexicon and + * lemmatizer are; the reference implementations of both are.</p> + */ +@ThreadSafe +public final class LexicalExpander { + + /** How an expansion relates to the input term. */ + public enum Kind { + + /** A member of one of the term's own synsets. */ + SYNONYM, + + /** A lemma of an ancestor synset, {@link Expansion#depth()} steps up. */ + HYPERNYM, + + /** A lemma of a direct child synset. */ + HYPONYM + } + + /** + * One expansion of a term. + * + * @param term The expanded term, in the lexicon's written form (multiword terms contain + * spaces). + * @param kind How the term relates to the input. + * @param depth The relation distance: {@code 0} for synonyms, the number of hypernym steps + * for {@link Kind#HYPERNYM}, {@code 1} for hyponyms. + * @param senseRank The zero-based rank of the input sense this expansion came from. + * @param weight The heuristic weight in {@code (0, 1]}; higher is closer to the input term. + */ + public record Expansion(String term, Kind kind, int depth, int senseRank, double weight) { + } + + private final LexicalKnowledgeBase lexicon; + private final Lemmatizer lemmatizer; + private final int maxSenses; + private final int hypernymDepth; + private final boolean includeHyponyms; + private final int maxExpansions; + private final double senseDecay; + private final double depthDecay; + + private LexicalExpander(Builder builder) { + this.lexicon = builder.lexicon; + this.lemmatizer = builder.lemmatizer; + this.maxSenses = builder.maxSenses; + this.hypernymDepth = builder.hypernymDepth; + this.includeHyponyms = builder.includeHyponyms; + this.maxExpansions = builder.maxExpansions; + this.senseDecay = builder.senseDecay; + this.depthDecay = builder.depthDecay; + } + + /** + * Starts a builder. + * + * @param lexicon The knowledge base to expand against; must not be null. + * @return A builder with the default configuration. + * @throws IllegalArgumentException Thrown if {@code lexicon} is null. + */ + public static Builder builder(LexicalKnowledgeBase lexicon) { + return new Builder(lexicon); + } + + /** + * Expands a term for one part of speech. + * + * @param term The term to expand; must not be null or blank. + * @param pos The part of speech to expand as; must not be null. + * @return The expansions, deduplicated and ordered by descending weight; empty when the term + * (and its lemma, when a lemmatizer is configured) is not in the lexicon. + * @throws IllegalArgumentException Thrown if {@code term} is null or blank or {@code pos} is + * null. + */ + public List<Expansion> expand(String term, WordNetPOS pos) { + if (pos == null) { + throw new IllegalArgumentException("The pos must not be null."); + } + return collect(term, List.of(pos)); + } + + /** + * Expands a term across all parts of speech. + * + * @param term The term to expand; must not be null or blank. + * @return The expansions across every part of speech, deduplicated and ordered by descending + * weight; empty when the term is not in the lexicon. + * @throws IllegalArgumentException Thrown if {@code term} is null or blank. + */ + public List<Expansion> expand(String term) { + return collect(term, List.of(WordNetPOS.values())); + } + + private List<Expansion> collect(String term, List<WordNetPOS> poses) { + if (term == null || term.isBlank()) { + throw new IllegalArgumentException("The term must not be null or blank."); + } + final Map<String, Expansion> best = new HashMap<>(); + final Set<String> excluded = new HashSet<>(); + excluded.add(fold(term)); + + for (final WordNetPOS pos : poses) { + final String subject = resolveSubject(term, pos); + if (subject == null) { + continue; + } + final List<Synset> senses = lexicon.lookup(subject, pos); + final int senseCount = Math.min(senses.size(), maxSenses); + for (int rank = 0; rank < senseCount; rank++) { + final double senseWeight = Math.pow(senseDecay, rank); + expandSense(senses.get(rank), rank, senseWeight, best, excluded); + } + } + + final List<Expansion> ordered = new ArrayList<>(best.values()); + ordered.sort(Comparator.comparingDouble(Expansion::weight).reversed() + .thenComparing(Expansion::kind) + .thenComparing(Expansion::term)); + return ordered.size() > maxExpansions ? List.copyOf(ordered.subList(0, maxExpansions)) + : List.copyOf(ordered); + } + + // The form actually expanded: the term when the lexicon knows it, otherwise its lemma when a + // lemmatizer is configured and produces a known lemma, otherwise nothing. + private String resolveSubject(String term, WordNetPOS pos) { + if (lexicon.contains(term, pos)) { + return term; + } + if (lemmatizer == null) { + return null; + } + final String[] lemmas = + lemmatizer.lemmatize(new String[] {term}, new String[] {pos.name()}); + if (lemmas.length == 0 || lemmas[0] == null) { + return null; + } + final String lemma = lemmas[0]; + // "O" is the Lemmatizer contract's unknown marker. + if ("O".equals(lemma) || !lexicon.contains(lemma, pos)) { + return null; + } + return lemma; + } + + private void expandSense(Synset sense, int rank, double senseWeight, + Map<String, Expansion> best, Set<String> excluded) { + for (final String lemma : sense.lemmas()) { + offer(best, excluded, new Expansion(lemma, Kind.SYNONYM, 0, rank, senseWeight)); + } + + // Breadth-first hypernym walk, visited-checked so cyclic data terminates. + if (hypernymDepth > 0) { + final Set<String> visited = new HashSet<>(); + visited.add(sense.id()); + final ArrayDeque<String> frontier = new ArrayDeque<>(hypernymsOf(sense)); + final ArrayDeque<String> next = new ArrayDeque<>(); + for (int depth = 1; depth <= hypernymDepth && !frontier.isEmpty(); depth++) { + final double depthWeight = senseWeight * Math.pow(depthDecay, depth); + while (!frontier.isEmpty()) { + final String id = frontier.poll(); + if (!visited.add(id)) { + continue; + } + final Synset ancestor = lexicon.synset(id).orElse(null); + if (ancestor == null) { + continue; + } + for (final String lemma : ancestor.lemmas()) { + offer(best, excluded, + new Expansion(lemma, Kind.HYPERNYM, depth, rank, depthWeight)); + } + next.addAll(hypernymsOf(ancestor)); + } + frontier.addAll(next); + next.clear(); + } + } + + if (includeHyponyms) { + final double hyponymWeight = senseWeight * depthDecay; + final List<String> children = new ArrayList<>(sense.related(WordNetRelation.HYPONYM)); + children.addAll(sense.related(WordNetRelation.INSTANCE_HYPONYM)); + for (final String id : children) { + lexicon.synset(id).ifPresent(child -> { + for (final String lemma : child.lemmas()) { + offer(best, excluded, new Expansion(lemma, Kind.HYPONYM, 1, rank, hyponymWeight)); + } + }); + } + } + } + + private static List<String> hypernymsOf(Synset synset) { + final List<String> direct = synset.related(WordNetRelation.HYPERNYM); + final List<String> instance = synset.related(WordNetRelation.INSTANCE_HYPERNYM); + if (instance.isEmpty()) { + return direct; + } + final List<String> all = new ArrayList<>(direct.size() + instance.size()); + all.addAll(direct); + all.addAll(instance); + return all; + } + + private static void offer(Map<String, Expansion> best, Set<String> excluded, + Expansion candidate) { + final String key = fold(candidate.term()); + if (excluded.contains(key)) { + return; + } + final Expansion current = best.get(key); + if (current == null || candidate.weight() > current.weight()) { + best.put(key, candidate); + } + } + + private static String fold(String term) { + return term.toLowerCase(Locale.ROOT); + } + + /** Configures and creates a {@link LexicalExpander}. */ + public static final class Builder { + + private final LexicalKnowledgeBase lexicon; + private Lemmatizer lemmatizer; + private int maxSenses = 3; + private int hypernymDepth = 1; + private boolean includeHyponyms = false; + private int maxExpansions = 20; + private double senseDecay = 0.5; + private double depthDecay = 0.5; + + private Builder(LexicalKnowledgeBase lexicon) { + if (lexicon == null) { + throw new IllegalArgumentException("The lexicon must not be null."); + } + this.lexicon = lexicon; + } + + /** + * Configures a lemmatizer used when the input term itself is not in the lexicon. It is + * invoked with the {@link WordNetPOS} name as the tag. + * + * @param lemmatizer The fallback lemmatizer; must not be null. + * @return This builder. + * @throws IllegalArgumentException Thrown if {@code lemmatizer} is null. + */ + public Builder lemmatizer(Lemmatizer lemmatizer) { + if (lemmatizer == null) { + throw new IllegalArgumentException("The lemmatizer must not be null."); + } + this.lemmatizer = lemmatizer; + return this; + } + + /** + * Configures how many senses of the term are expanded, most salient first. + * + * @param maxSenses The sense count; must be positive. The default is {@code 3}. + * @return This builder. + * @throws IllegalArgumentException Thrown if {@code maxSenses} is not positive. + */ + public Builder maxSenses(int maxSenses) { + if (maxSenses < 1) { + throw new IllegalArgumentException("The maxSenses must be positive: " + maxSenses); + } + this.maxSenses = maxSenses; + return this; + } + + /** + * Configures how many hypernym steps are walked; {@code 0} disables hypernym expansion. + * + * @param hypernymDepth The depth; must not be negative. The default is {@code 1}. + * @return This builder. + * @throws IllegalArgumentException Thrown if {@code hypernymDepth} is negative. + */ + public Builder hypernymDepth(int hypernymDepth) { + if (hypernymDepth < 0) { + throw new IllegalArgumentException( + "The hypernymDepth must not be negative: " + hypernymDepth); + } + this.hypernymDepth = hypernymDepth; + return this; + } + + /** + * Configures whether direct hyponyms are included. Hyponyms narrow rather than generalize, + * which suits some matching tasks and hurts others, so they are off by default. + * + * @param includeHyponyms Whether to include direct hyponyms. + * @return This builder. + */ + public Builder includeHyponyms(boolean includeHyponyms) { + this.includeHyponyms = includeHyponyms; + return this; + } + + /** + * Configures the maximum number of expansions returned after ranking. + * + * @param maxExpansions The cap; must be positive. The default is {@code 20}. + * @return This builder. + * @throws IllegalArgumentException Thrown if {@code maxExpansions} is not positive. + */ + public Builder maxExpansions(int maxExpansions) { + if (maxExpansions < 1) { + throw new IllegalArgumentException( + "The maxExpansions must be positive: " + maxExpansions); + } + this.maxExpansions = maxExpansions; + return this; + } + + /** + * Configures the weight multiplier applied per sense rank step. + * + * @param senseDecay The decay in {@code (0, 1]}. The default is {@code 0.5}. + * @return This builder. + * @throws IllegalArgumentException Thrown if {@code senseDecay} is outside {@code (0, 1]}. + */ + public Builder senseDecay(double senseDecay) { + if (!(senseDecay > 0 && senseDecay <= 1)) { + throw new IllegalArgumentException( + "The senseDecay must be in (0, 1]: " + senseDecay); + } + this.senseDecay = senseDecay; + return this; + } + + /** + * Configures the weight multiplier applied per hypernym or hyponym step. + * + * @param depthDecay The decay in {@code (0, 1]}. The default is {@code 0.5}. + * @return This builder. + * @throws IllegalArgumentException Thrown if {@code depthDecay} is outside {@code (0, 1]}. + */ + public Builder depthDecay(double depthDecay) { + if (!(depthDecay > 0 && depthDecay <= 1)) { + throw new IllegalArgumentException( + "The depthDecay must be in (0, 1]: " + depthDecay); + } + this.depthDecay = depthDecay; + return this; + } + + /** {@return the configured expander} */ + public LexicalExpander build() { + return new LexicalExpander(this); + } + } +} diff --git a/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/LexicalExpanderLexiconTest.java b/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/LexicalExpanderLexiconTest.java new file mode 100644 index 000000000..44506a1d5 --- /dev/null +++ b/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/LexicalExpanderLexiconTest.java @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.wordnet; + +import java.util.List; + +import org.junit.jupiter.api.Test; + +import opennlp.tools.wordnet.LexicalKnowledgeBase; +import opennlp.tools.wordnet.WordNetPOS; +import opennlp.wordnet.LexicalExpander.Expansion; +import opennlp.wordnet.LexicalExpander.Kind; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * End-to-end expansion over the miniature lexicon fixtures: the WN-LMF and WNDB readers each + * feed the expander, and the Morphy lemmatizer bridges inflected input, exercising the whole + * stack the way a consumer wires it. + */ +class LexicalExpanderLexiconTest { + + private static Expansion find(List<Expansion> expansions, String term) { + return expansions.stream().filter(e -> e.term().equals(term)).findFirst().orElse(null); + } + + @Test + void testExpansionOverTheWnLmfLexicon() { + final LexicalExpander expander = + LexicalExpander.builder(WnLmfReaderTest.fixture()).build(); + + final List<Expansion> expansions = expander.expand("dog", WordNetPOS.NOUN); + final Expansion domesticDog = find(expansions, "domestic dog"); + assertEquals(Kind.SYNONYM, domesticDog.kind()); + assertEquals(1.0, domesticDog.weight()); + final Expansion canid = find(expansions, "canid"); + assertEquals(Kind.HYPERNYM, canid.kind()); + assertEquals(0.5, canid.weight()); + } + + @Test + void testExpansionOverTheWndbLexicon() { + final LexicalExpander expander = + LexicalExpander.builder(WndbReaderTest.fixture()).build(); + + final List<Expansion> expansions = expander.expand("dog", WordNetPOS.NOUN); + assertTrue(find(expansions, "domestic dog") != null, "got " + expansions); + final Expansion canid = find(expansions, "canid"); + assertEquals(Kind.HYPERNYM, canid.kind()); + } + + @Test + void testReadersAgreeOnExpansions() { + final List<Expansion> lmf = LexicalExpander.builder(WnLmfReaderTest.fixture()) + .hypernymDepth(2).build().expand("mouse", WordNetPOS.NOUN); + final List<Expansion> wndb = LexicalExpander.builder(WndbReaderTest.fixture()) + .hypernymDepth(2).build().expand("mouse", WordNetPOS.NOUN); + + assertEquals( + lmf.stream().map(e -> e.term() + "|" + e.kind() + "|" + e.weight()).toList(), + wndb.stream().map(e -> e.term() + "|" + e.kind() + "|" + e.weight()).toList()); + assertTrue(find(lmf, "rodent") != null, "got " + lmf); + } + + @Test + void testMorphyBridgesInflectedInput() { + final LexicalKnowledgeBase lexicon = WnLmfReaderTest.fixture(); + final LexicalExpander expander = LexicalExpander.builder(lexicon) + .lemmatizer(new MorphyLemmatizer(lexicon, MorphyExceptionsTest.fixture())) + .build(); + + // A regular inflection resolves by rule, an irregular one by the exception list. + final List<Expansion> dogs = expander.expand("dogs", WordNetPOS.NOUN); + assertEquals(Kind.SYNONYM, find(dogs, "dog").kind()); + assertTrue(find(dogs, "canid") != null, "got " + dogs); + + final List<Expansion> mice = expander.expand("mice", WordNetPOS.NOUN); + assertEquals(Kind.SYNONYM, find(mice, "mouse").kind()); + assertTrue(find(mice, "rodent") != null, "got " + mice); + } +} diff --git a/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/LexicalExpanderTest.java b/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/LexicalExpanderTest.java new file mode 100644 index 000000000..9d30490c1 --- /dev/null +++ b/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/LexicalExpanderTest.java @@ -0,0 +1,270 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.wordnet; + +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Optional; + +import org.junit.jupiter.api.Test; + +import opennlp.tools.wordnet.LexicalKnowledgeBase; +import opennlp.tools.wordnet.Synset; +import opennlp.tools.wordnet.WordNetPOS; +import opennlp.tools.wordnet.WordNetRelation; +import opennlp.wordnet.LexicalExpander.Expansion; +import opennlp.wordnet.LexicalExpander.Kind; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Behavioral tests over a hand-built lexicon whose graph shape is fully controlled: sense + * ranking, hypernym depth and decay, hyponym opt-in, deduplication, exclusion of the input, + * cycle termination, and configuration validation. + */ +class LexicalExpanderTest { + + // dog: sense 1 = {dog, domestic dog} -> canid -> carnivore, with hyponym puppy; + // sense 2 = {dog, frank, hot dog} -> sausage. The verb sense = {dog, chase}. + // alpha <-> beta form a malformed hypernym cycle. + private static LexicalKnowledgeBase lexicon() { + final Map<String, Synset> synsets = new HashMap<>(); + final Map<String, List<Synset>> senses = new HashMap<>(); + + final Synset n1 = new Synset("n1", WordNetPOS.NOUN, List.of("dog", "domestic dog"), "canine", + Map.of(WordNetRelation.HYPERNYM, List.of("n2"), + WordNetRelation.HYPONYM, List.of("n4"))); + final Synset n2 = new Synset("n2", WordNetPOS.NOUN, List.of("canid"), "canid family", + Map.of(WordNetRelation.HYPERNYM, List.of("n3"))); + final Synset n3 = new Synset("n3", WordNetPOS.NOUN, List.of("carnivore"), "meat eater", + Map.of()); + final Synset n4 = new Synset("n4", WordNetPOS.NOUN, List.of("puppy"), "young dog", Map.of()); + final Synset n5 = new Synset("n5", WordNetPOS.NOUN, List.of("dog", "frank", "hot dog"), + "sausage in a bun", Map.of(WordNetRelation.HYPERNYM, List.of("n6"))); + final Synset n6 = new Synset("n6", WordNetPOS.NOUN, List.of("sausage"), "ground meat", + Map.of()); + final Synset v1 = new Synset("v1", WordNetPOS.VERB, List.of("dog", "chase"), "follow", + Map.of()); + final Synset c1 = new Synset("c1", WordNetPOS.NOUN, List.of("alpha"), "cycle start", + Map.of(WordNetRelation.HYPERNYM, List.of("c2"))); + final Synset c2 = new Synset("c2", WordNetPOS.NOUN, List.of("beta"), "cycle end", + Map.of(WordNetRelation.HYPERNYM, List.of("c1"))); + + for (final Synset synset : List.of(n1, n2, n3, n4, n5, n6, v1, c1, c2)) { + synsets.put(synset.id(), synset); + } + senses.put("dog|NOUN", List.of(n1, n5)); + senses.put("dog|VERB", List.of(v1)); + senses.put("domestic dog|NOUN", List.of(n1)); + senses.put("alpha|NOUN", List.of(c1)); + + return new LexicalKnowledgeBase() { + @Override + public List<Synset> lookup(String lemma, WordNetPOS pos) { + if (lemma == null || pos == null) { + throw new IllegalArgumentException("null"); + } + return senses.getOrDefault(lemma.toLowerCase(Locale.ROOT) + "|" + pos, List.of()); + } + + @Override + public Optional<Synset> synset(String synsetId) { + return Optional.ofNullable(synsets.get(synsetId)); + } + }; + } + + private static Expansion find(List<Expansion> expansions, String term) { + return expansions.stream().filter(e -> e.term().equals(term)).findFirst().orElse(null); + } + + @Test + void testSynonymsAndHypernymsWithDefaultConfiguration() { + final List<Expansion> expansions = + LexicalExpander.builder(lexicon()).build().expand("dog", WordNetPOS.NOUN); + + final Expansion domesticDog = find(expansions, "domestic dog"); + assertEquals(Kind.SYNONYM, domesticDog.kind()); + assertEquals(1.0, domesticDog.weight()); + assertEquals(0, domesticDog.senseRank()); + + final Expansion canid = find(expansions, "canid"); + assertEquals(Kind.HYPERNYM, canid.kind()); + assertEquals(1, canid.depth()); + assertEquals(0.5, canid.weight()); + + final Expansion frank = find(expansions, "frank"); + assertEquals(Kind.SYNONYM, frank.kind()); + assertEquals(1, frank.senseRank()); + assertEquals(0.5, frank.weight()); + + // Depth 1 by default: the grandparent stays out, and so do hyponyms. + assertEquals(null, find(expansions, "carnivore")); + assertEquals(null, find(expansions, "puppy")); + } + + @Test + void testTheInputTermIsNeverAnExpansion() { + for (final Expansion expansion : + LexicalExpander.builder(lexicon()).build().expand("dog", WordNetPOS.NOUN)) { + assertTrue(!expansion.term().equalsIgnoreCase("dog"), "got " + expansion); + } + } + + @Test + void testDeeperHypernymWalkDecaysPerStep() { + final List<Expansion> expansions = LexicalExpander.builder(lexicon()) + .hypernymDepth(2).build().expand("dog", WordNetPOS.NOUN); + + final Expansion carnivore = find(expansions, "carnivore"); + assertEquals(2, carnivore.depth()); + assertEquals(0.25, carnivore.weight()); + } + + @Test + void testHyponymsAreOptIn() { + final List<Expansion> expansions = LexicalExpander.builder(lexicon()) + .includeHyponyms(true).build().expand("dog", WordNetPOS.NOUN); + + final Expansion puppy = find(expansions, "puppy"); + assertEquals(Kind.HYPONYM, puppy.kind()); + assertEquals(0.5, puppy.weight()); + } + + @Test + void testMaxSensesLimitsToTheMostSalient() { + final List<Expansion> expansions = LexicalExpander.builder(lexicon()) + .maxSenses(1).build().expand("dog", WordNetPOS.NOUN); + + assertEquals(null, find(expansions, "frank")); + assertTrue(find(expansions, "domestic dog") != null); + } + + @Test + void testAllPosExpansionIncludesVerbSynonyms() { + final List<Expansion> expansions = + LexicalExpander.builder(lexicon()).build().expand("dog"); + + assertTrue(find(expansions, "chase") != null); + assertTrue(find(expansions, "domestic dog") != null); + } + + @Test + void testCyclicHypernymDataTerminates() { + final List<Expansion> expansions = LexicalExpander.builder(lexicon()) + .hypernymDepth(10).build().expand("alpha", WordNetPOS.NOUN); + + final Expansion beta = find(expansions, "beta"); + assertEquals(1, beta.depth()); + // The cycle leads back to alpha's own synset, which is visited and the input besides. + assertEquals(1, expansions.size()); + } + + @Test + void testDeduplicationKeepsTheHighestWeight() { + // "domestic dog" reaches n1 at rank 0; its synonym "dog" is the only other member and + // must appear once with the rank-0 weight even though deeper paths could yield it again. + final List<Expansion> expansions = LexicalExpander.builder(lexicon()) + .hypernymDepth(2).build().expand("domestic dog", WordNetPOS.NOUN); + + final Expansion dog = find(expansions, "dog"); + assertEquals(1.0, dog.weight()); + assertEquals(1, expansions.stream().filter(e -> e.term().equals("dog")).count()); + } + + @Test + void testOrderingIsWeightDescendingAndStable() { + final List<Expansion> expansions = LexicalExpander.builder(lexicon()) + .hypernymDepth(2).includeHyponyms(true).build().expand("dog", WordNetPOS.NOUN); + + for (int i = 1; i < expansions.size(); i++) { + assertTrue(expansions.get(i - 1).weight() >= expansions.get(i).weight(), + "weights must not increase: " + expansions); + } + assertEquals(expansions, + LexicalExpander.builder(lexicon()).hypernymDepth(2).includeHyponyms(true).build() + .expand("dog", WordNetPOS.NOUN)); + } + + @Test + void testMaxExpansionsCapsAfterRanking() { + final List<Expansion> expansions = LexicalExpander.builder(lexicon()) + .hypernymDepth(2).includeHyponyms(true).maxExpansions(2).build() + .expand("dog", WordNetPOS.NOUN); + + assertEquals(2, expansions.size()); + assertEquals(1.0, expansions.get(0).weight()); + } + + @Test + void testUnknownTermExpandsToNothing() { + assertEquals(List.of(), + LexicalExpander.builder(lexicon()).build().expand("xyzzy", WordNetPOS.NOUN)); + } + + @Test + void testLemmatizerFallbackExpandsInflectedInput() { + final LexicalExpander expander = LexicalExpander.builder(lexicon()) + .lemmatizer(new opennlp.tools.lemmatizer.Lemmatizer() { + @Override + public String[] lemmatize(String[] tokens, String[] tags) { + final String[] lemmas = new String[tokens.length]; + for (int i = 0; i < tokens.length; i++) { + lemmas[i] = "dogs".equals(tokens[i]) ? "dog" : "O"; + } + return lemmas; + } + + @Override + public List<List<String>> lemmatize(List<String> tokens, List<String> tags) { + throw new UnsupportedOperationException(); + } + }) + .build(); + + final List<Expansion> expansions = expander.expand("dogs", WordNetPOS.NOUN); + // The lemma itself surfaces as a synonym, along with the rest of its synsets. + final Expansion dog = find(expansions, "dog"); + assertEquals(Kind.SYNONYM, dog.kind()); + assertEquals(1.0, dog.weight()); + assertTrue(find(expansions, "domestic dog") != null); + + assertEquals(List.of(), expander.expand("cats", WordNetPOS.NOUN)); + } + + @Test + void testValidationFailsLoudly() { + assertThrows(IllegalArgumentException.class, () -> LexicalExpander.builder(null)); + final LexicalExpander.Builder builder = LexicalExpander.builder(lexicon()); + assertThrows(IllegalArgumentException.class, () -> builder.lemmatizer(null)); + assertThrows(IllegalArgumentException.class, () -> builder.maxSenses(0)); + assertThrows(IllegalArgumentException.class, () -> builder.hypernymDepth(-1)); + assertThrows(IllegalArgumentException.class, () -> builder.maxExpansions(0)); + assertThrows(IllegalArgumentException.class, () -> builder.senseDecay(0)); + assertThrows(IllegalArgumentException.class, () -> builder.senseDecay(1.5)); + assertThrows(IllegalArgumentException.class, () -> builder.depthDecay(0)); + + final LexicalExpander expander = builder.build(); + assertThrows(IllegalArgumentException.class, () -> expander.expand(null)); + assertThrows(IllegalArgumentException.class, () -> expander.expand(" ")); + assertThrows(IllegalArgumentException.class, () -> expander.expand("dog", null)); + } +}
