This is an automated email from the ASF dual-hosted git repository. krickert pushed a commit to branch OPENNLP-1870 in repository https://gitbox.apache.org/repos/asf/opennlp.git
commit e43943b998898f306019fef1106e69e04101e618 Author: Kristian Rickert <[email protected]> AuthorDate: Mon Jul 6 22:14:59 2026 -0400 OPENNLP-1870: Wire emoji annotations as opt-in features for NameFinder, Doccat, and Sentiment Wires the annotation layer into the three components named by the epic, each through its existing extension seam, so default behavior and existing models are unchanged. - Name finder (and every consumer of feature generation descriptors): EmojiAnnotationFeatureGenerator (AdaptiveFeatureGenerator) emits emoji.sentiment/emoji.type/emoji.category/emoji.region features for the indexed token; the entity type behaves like gazetteer evidence (a flag token carries emoji.type=FLAG and its region with no dictionary). Activated only when a descriptor names the new EmojiAnnotationFeatureGeneratorFactory; no default descriptor changes. - Document categorizer: EmojiFeatureGenerator (doccat FeatureGenerator) emits emojiSentiment/emojiEntityType/emojiCategory/emojiRegion document features; opt-in through the existing DoccatFactory feature generator array, whose default (BagOfWords only) is untouched. The no-arg constructor lets a trained model re-instantiate it from its manifest. - Sentiment: EmojiSentimentContextGenerator appends the same evidence to the default token context, and EmojiSentimentFactory is the opt-in training seam; the factory class is recorded in the model manifest, so prediction re-creates the identical context (verified against SentimentME's factory.createContextGenerator() path). Without emoji in the text its context is byte-identical to the default generator's. - All three generators fast-path ASCII-leading tokens without touching the annotation layer; a new audit in EmojiAnnotationsTest pins that every annotatable symbol starts beyond ASCII so the guard cannot drift. - Bulk safety proven by test: a lone regional indicator in damaged text contributes nothing instead of throwing. - Manual: wiring paragraph in the Emoji annotations section. Part of the OPENNLP-1852 epic. --- .../tools/doccat/EmojiFeatureGenerator.java | 95 ++++++++++++++++ .../sentiment/EmojiSentimentContextGenerator.java | 74 ++++++++++++ .../tools/sentiment/EmojiSentimentFactory.java | 38 +++++++ .../EmojiAnnotationFeatureGenerator.java | 90 +++++++++++++++ .../EmojiAnnotationFeatureGeneratorFactory.java | 43 +++++++ .../tools/doccat/EmojiFeatureGeneratorTest.java | 87 +++++++++++++++ .../EmojiSentimentContextGeneratorTest.java | 71 ++++++++++++ .../EmojiAnnotationFeatureGeneratorTest.java | 124 +++++++++++++++++++++ .../util/normalizer/EmojiAnnotationsTest.java | 11 ++ opennlp-docs/src/docbkx/normalizer.xml | 13 +++ 10 files changed, 646 insertions(+) diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/doccat/EmojiFeatureGenerator.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/doccat/EmojiFeatureGenerator.java new file mode 100644 index 000000000..6373d56e4 --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/doccat/EmojiFeatureGenerator.java @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.doccat; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Map; + +import opennlp.tools.util.normalizer.EmojiAnnotation; +import opennlp.tools.util.normalizer.EmojiAnnotator; + +/** + * Generates document features from the emoji annotation layer: for every annotated token the + * project-authored coarse sentiment score ({@code emojiSentiment=2}), the entity type + * ({@code emojiEntityType=HEART}), the document category hint + * ({@code emojiCategory=SMILEYS_AND_EMOTION}), and for flags the ISO 3166 region + * ({@code emojiRegion=DE}). Tokens that are not annotated symbols contribute nothing. + * + * <p>Strictly opt-in: pass it alongside the defaults, for example + * {@code new DoccatFactory(new FeatureGenerator[] {new BagOfWordsFeatureGenerator(), + * new EmojiFeatureGenerator()})}; the default factory configuration is unchanged. The no-argument + * constructor is required so a trained model can re-instantiate the generator from its + * manifest.</p> + * + * @see EmojiAnnotator + */ +public class EmojiFeatureGenerator implements FeatureGenerator { + + private static final String SENTIMENT_PREFIX = "emojiSentiment="; + private static final String ENTITY_TYPE_PREFIX = "emojiEntityType="; + private static final String CATEGORY_PREFIX = "emojiCategory="; + private static final String REGION_PREFIX = "emojiRegion="; + + private final EmojiAnnotator annotator; + + /** + * Instantiates a generator over the bundled and derived annotation layers. + */ + public EmojiFeatureGenerator() { + this(new EmojiAnnotator()); + } + + /** + * Instantiates a generator over a configured annotator, for example one with a gazetteer join. + * + * @param annotator The annotator to use. Must not be {@code null}. + * @throws IllegalArgumentException if {@code annotator} is {@code null}. + */ + public EmojiFeatureGenerator(EmojiAnnotator annotator) { + if (annotator == null) { + throw new IllegalArgumentException("Annotator must not be null"); + } + this.annotator = annotator; + } + + @Override + public Collection<String> extractFeatures(String[] text, Map<String, Object> extraInformation) { + if (text == null) { + throw new IllegalArgumentException("Text must not be null"); + } + final List<String> features = new ArrayList<>(); + for (final String token : text) { + // Every annotatable symbol starts beyond ASCII (audited in EmojiAnnotationsTest), so + // ordinary tokens exit here without touching the annotation layer. + if (token.isEmpty() || token.charAt(0) < 0x80) { + continue; + } + final EmojiAnnotation annotation = annotator.annotate(token).orElse(null); + if (annotation == null) { + continue; + } + annotation.sentiment().ifPresent(score -> features.add(SENTIMENT_PREFIX + score)); + annotation.entityType().ifPresent(type -> features.add(ENTITY_TYPE_PREFIX + type)); + annotation.category().ifPresent(category -> features.add(CATEGORY_PREFIX + category)); + annotation.isoRegion().ifPresent(region -> features.add(REGION_PREFIX + region)); + } + return features; + } +} diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/sentiment/EmojiSentimentContextGenerator.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/sentiment/EmojiSentimentContextGenerator.java new file mode 100644 index 000000000..ba174fddd --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/sentiment/EmojiSentimentContextGenerator.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.sentiment; + +import java.util.ArrayList; +import java.util.List; + +import opennlp.tools.util.normalizer.EmojiAnnotation; +import opennlp.tools.util.normalizer.EmojiAnnotator; + +/** + * A {@link SentimentContextGenerator} that adds emoji annotation features to the default + * token context: for every annotated token the project-authored coarse sentiment score + * ({@code emojiSentiment=2}), the entity type ({@code emojiEntityType=HEART}), the category + * ({@code emojiCategory=SMILEYS_AND_EMOTION}), and for flags the region ({@code emojiRegion=DE}). + * A crying-face or heart pictograph thereby becomes direct evidence for the sentiment model + * instead of an opaque token. + * + * <p>Strictly opt-in through the {@link SentimentFactory} seam: train with + * {@link EmojiSentimentFactory} (or any factory whose + * {@link SentimentFactory#createContextGenerator()} returns this class) and the same context is + * regenerated at prediction time; the default factory and existing models are unchanged.</p> + * + * @see EmojiSentimentFactory + * @see EmojiAnnotator + */ +public class EmojiSentimentContextGenerator extends SentimentContextGenerator { + + private static final String SENTIMENT_PREFIX = "emojiSentiment="; + private static final String ENTITY_TYPE_PREFIX = "emojiEntityType="; + private static final String CATEGORY_PREFIX = "emojiCategory="; + private static final String REGION_PREFIX = "emojiRegion="; + + private final EmojiAnnotator annotator = new EmojiAnnotator(); + + @Override + public String[] getContext(String[] text) { + final List<String> context = new ArrayList<>(text.length); + for (final String token : text) { + context.add(token); // the default context: every token is a feature + } + for (final String token : text) { + // Every annotatable symbol starts beyond ASCII (audited in EmojiAnnotationsTest), so + // ordinary tokens exit here without touching the annotation layer. + if (token.isEmpty() || token.charAt(0) < 0x80) { + continue; + } + final EmojiAnnotation annotation = annotator.annotate(token).orElse(null); + if (annotation == null) { + continue; + } + annotation.sentiment().ifPresent(score -> context.add(SENTIMENT_PREFIX + score)); + annotation.entityType().ifPresent(type -> context.add(ENTITY_TYPE_PREFIX + type)); + annotation.category().ifPresent(category -> context.add(CATEGORY_PREFIX + category)); + annotation.isoRegion().ifPresent(region -> context.add(REGION_PREFIX + region)); + } + return context.toArray(new String[0]); + } +} diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/sentiment/EmojiSentimentFactory.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/sentiment/EmojiSentimentFactory.java new file mode 100644 index 000000000..3c4cb1f05 --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/sentiment/EmojiSentimentFactory.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.sentiment; + +/** + * A {@link SentimentFactory} whose context generator adds emoji annotation features (see + * {@link EmojiSentimentContextGenerator}). Pass it to + * {@link SentimentME#train(String, opennlp.tools.util.ObjectStream, + * opennlp.tools.util.TrainingParameters, SentimentFactory) SentimentME.train} to opt in; the + * factory class is recorded in the trained model's manifest, so prediction re-creates the same + * context. The default {@link SentimentFactory} is unchanged. + */ +public class EmojiSentimentFactory extends SentimentFactory { + + public EmojiSentimentFactory() { + super(); + } + + @Override + public SentimentContextGenerator createContextGenerator() { + return new EmojiSentimentContextGenerator(); + } +} diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/featuregen/EmojiAnnotationFeatureGenerator.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/featuregen/EmojiAnnotationFeatureGenerator.java new file mode 100644 index 000000000..970e06e0e --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/featuregen/EmojiAnnotationFeatureGenerator.java @@ -0,0 +1,90 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.util.featuregen; + +import java.util.List; + +import opennlp.tools.util.normalizer.EmojiAnnotation; +import opennlp.tools.util.normalizer.EmojiAnnotator; + +/** + * Generates features from the emoji annotation layer for the token at the given index: the + * project-authored coarse sentiment score ({@code emoji.sentiment=2}), the entity type + * ({@code emoji.type=HEART}), the document category ({@code emoji.category=SMILEYS_AND_EMOTION}), + * and for flags the ISO 3166 region ({@code emoji.region=DE}). For the name finder the entity + * type behaves like gazetteer evidence: a flag token carries {@code emoji.type=FLAG} and its + * region without any dictionary. + * + * <p>Strictly opt-in: this generator only runs when a feature generation descriptor names + * {@link EmojiAnnotationFeatureGeneratorFactory}, so default descriptors and existing models are + * unchanged. A token that is not an annotated symbol contributes no features.</p> + * + * <p>Unlike most feature generators this one is stateless and thread-safe (when its annotator's + * join is): it keeps no adaptive data.</p> + * + * @see EmojiAnnotator + */ +public class EmojiAnnotationFeatureGenerator implements AdaptiveFeatureGenerator { + + private static final String SENTIMENT_PREFIX = "emoji.sentiment="; + private static final String TYPE_PREFIX = "emoji.type="; + private static final String CATEGORY_PREFIX = "emoji.category="; + private static final String REGION_PREFIX = "emoji.region="; + + private final EmojiAnnotator annotator; + + /** + * Instantiates a generator over the bundled and derived annotation layers. + */ + public EmojiAnnotationFeatureGenerator() { + this(new EmojiAnnotator()); + } + + /** + * Instantiates a generator over a configured annotator, for example one with a gazetteer join. + * + * @param annotator The annotator to use. Must not be {@code null}. + * @throws IllegalArgumentException if {@code annotator} is {@code null}. + */ + public EmojiAnnotationFeatureGenerator(EmojiAnnotator annotator) { + if (annotator == null) { + throw new IllegalArgumentException("Annotator must not be null"); + } + this.annotator = annotator; + } + + @Override + public void createFeatures(List<String> features, String[] tokens, int index, + String[] previousOutcomes) { + final String token = tokens[index]; + // Every annotatable symbol starts beyond ASCII (a pictograph, a regional indicator, or the + // waving black flag; audited in EmojiAnnotationsTest), so ordinary tokens exit here without + // touching the annotation layer. + if (token.isEmpty() || token.charAt(0) < 0x80) { + return; + } + final EmojiAnnotation annotation = annotator.annotate(token).orElse(null); + if (annotation == null) { + return; + } + annotation.sentiment().ifPresent(score -> features.add(SENTIMENT_PREFIX + score)); + annotation.entityType().ifPresent(type -> features.add(TYPE_PREFIX + type)); + annotation.category().ifPresent(category -> features.add(CATEGORY_PREFIX + category)); + annotation.isoRegion().ifPresent(region -> features.add(REGION_PREFIX + region)); + } +} diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/featuregen/EmojiAnnotationFeatureGeneratorFactory.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/featuregen/EmojiAnnotationFeatureGeneratorFactory.java new file mode 100644 index 000000000..ddd2992eb --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/featuregen/EmojiAnnotationFeatureGeneratorFactory.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.util.featuregen; + +import opennlp.tools.util.InvalidFormatException; + +/** + * A {@link GeneratorFactory} that produces {@link EmojiAnnotationFeatureGenerator} instances + * when {@link #create()} is called, for use in a feature generation descriptor: + * + * <pre>{@code + * <generator class="opennlp.tools.util.featuregen.EmojiAnnotationFeatureGeneratorFactory"/> + * }</pre> + * + * @see EmojiAnnotationFeatureGenerator + */ +public class EmojiAnnotationFeatureGeneratorFactory + extends GeneratorFactory.AbstractXmlFeatureGeneratorFactory { + + public EmojiAnnotationFeatureGeneratorFactory() { + super(); + } + + @Override + public AdaptiveFeatureGenerator create() throws InvalidFormatException { + return new EmojiAnnotationFeatureGenerator(); + } +} diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/doccat/EmojiFeatureGeneratorTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/doccat/EmojiFeatureGeneratorTest.java new file mode 100644 index 000000000..3957b8ac5 --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/doccat/EmojiFeatureGeneratorTest.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.doccat; + +import java.util.Collection; +import java.util.List; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class EmojiFeatureGeneratorTest { + + private static String cp(int... codePoints) { + final StringBuilder sb = new StringBuilder(); + for (final int codePoint : codePoints) { + sb.appendCodePoint(codePoint); + } + return sb.toString(); + } + + @Test + void annotatedTokensContributeDocumentFeatures() { + final FeatureGenerator generator = new EmojiFeatureGenerator(); + final Collection<String> features = generator.extractFeatures( + new String[] {"great", "pizza", cp(0x1F355), cp(0x1F600)}, null); + assertEquals(List.of( + "emojiSentiment=0", "emojiEntityType=FOOD", "emojiCategory=FOOD_AND_DRINK", + "emojiSentiment=2", "emojiEntityType=FACE", "emojiCategory=SMILEYS_AND_EMOTION"), + List.copyOf(features)); + } + + @Test + void flagsContributeTheirRegion() { + final FeatureGenerator generator = new EmojiFeatureGenerator(); + final Collection<String> features = generator.extractFeatures( + new String[] {"match", cp(0x1F1EB, 0x1F1F7)}, null); + assertTrue(features.contains("emojiRegion=FR"), "Missing region feature in: " + features); + assertTrue(features.contains("emojiEntityType=FLAG"), "Missing type feature in: " + features); + } + + @Test + void plainDocumentsContributeNothing() { + final FeatureGenerator generator = new EmojiFeatureGenerator(); + // Includes a lone regional indicator: bulk extraction must survive damaged text. + final Collection<String> features = generator.extractFeatures( + new String[] {"no", "emoji", "here", "", cp(0x1F1E9)}, null); + assertTrue(features.isEmpty(), "Unexpected features: " + features); + } + + @Test + void argumentsAreValidated() { + final FeatureGenerator generator = new EmojiFeatureGenerator(); + assertThrows(IllegalArgumentException.class, () -> generator.extractFeatures(null, null)); + assertThrows(IllegalArgumentException.class, () -> new EmojiFeatureGenerator(null)); + } + + @Test + void optInThroughTheDoccatFactorySeam() { + // The wiring is the existing DoccatFactory feature generator seam; the default factory keeps + // only BagOfWords, so default behavior is unchanged and this factory is the opt-in. + final DoccatFactory factory = new DoccatFactory(new FeatureGenerator[] { + new BagOfWordsFeatureGenerator(), new EmojiFeatureGenerator()}); + assertEquals(2, factory.getFeatureGenerators().length); + final DoccatFactory defaults = new DoccatFactory(); + assertEquals(1, defaults.getFeatureGenerators().length); + assertEquals(BagOfWordsFeatureGenerator.class, + defaults.getFeatureGenerators()[0].getClass()); + } +} diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/sentiment/EmojiSentimentContextGeneratorTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/sentiment/EmojiSentimentContextGeneratorTest.java new file mode 100644 index 000000000..184d0e9c3 --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/sentiment/EmojiSentimentContextGeneratorTest.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.sentiment; + +import java.util.List; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class EmojiSentimentContextGeneratorTest { + + private static String cp(int... codePoints) { + final StringBuilder sb = new StringBuilder(); + for (final int codePoint : codePoints) { + sb.appendCodePoint(codePoint); + } + return sb.toString(); + } + + @Test + void emojiFeaturesAreAppendedToTheTokenContext() { + final String[] tokens = {"so", "sad", cp(0x1F62D)}; + final String[] context = new EmojiSentimentContextGenerator().getContext(tokens); + final List<String> features = List.of(context); + // The default token context is preserved in full... + assertEquals(List.of(tokens), features.subList(0, tokens.length)); + // ...and the pictograph additionally contributes its typed evidence. + assertTrue(features.contains("emojiSentiment=-2"), "Missing feature in: " + features); + assertTrue(features.contains("emojiEntityType=FACE"), "Missing feature in: " + features); + assertTrue(features.contains("emojiCategory=SMILEYS_AND_EMOTION"), + "Missing feature in: " + features); + } + + @Test + void plainTextContextEqualsTheDefaultGenerators() { + // Without emoji the opt-in generator produces exactly the default context, so opting in + // cannot disturb what a model learns from ordinary text. + final String[] tokens = {"no", "emoji", "here", "", cp(0x1F1E9)}; + assertArrayEquals(new SentimentContextGenerator().getContext(tokens), + new EmojiSentimentContextGenerator().getContext(tokens)); + } + + @Test + void theFactorySeamCreatesTheOptInGenerator() { + // Opt-in path: EmojiSentimentFactory is passed to SentimentME.train and recorded in the + // model manifest, so prediction re-creates the same context generator. The default factory + // is unchanged. + assertEquals(EmojiSentimentContextGenerator.class, + new EmojiSentimentFactory().createContextGenerator().getClass()); + assertEquals(SentimentContextGenerator.class, + new SentimentFactory().createContextGenerator().getClass()); + } +} diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/featuregen/EmojiAnnotationFeatureGeneratorTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/featuregen/EmojiAnnotationFeatureGeneratorTest.java new file mode 100644 index 000000000..8dca486b6 --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/featuregen/EmojiAnnotationFeatureGeneratorTest.java @@ -0,0 +1,124 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.util.featuregen; + +import java.io.ByteArrayInputStream; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import org.junit.jupiter.api.Test; + +import opennlp.tools.util.normalizer.EmojiAnnotation; +import opennlp.tools.util.normalizer.EmojiAnnotator; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class EmojiAnnotationFeatureGeneratorTest { + + private static String cp(int... codePoints) { + final StringBuilder sb = new StringBuilder(); + for (final int codePoint : codePoints) { + sb.appendCodePoint(codePoint); + } + return sb.toString(); + } + + @Test + void annotatedTokensContributeTypedFeatures() { + final AdaptiveFeatureGenerator generator = new EmojiAnnotationFeatureGenerator(); + final String[] tokens = {"I", cp(0x2764, 0xFE0F), "Berlin", cp(0x1F1E9, 0x1F1EA)}; + final List<String> features = new ArrayList<>(); + generator.createFeatures(features, tokens, 1, null); + assertEquals(List.of("emoji.sentiment=2", "emoji.type=HEART", + "emoji.category=SMILEYS_AND_EMOTION"), features); + features.clear(); + generator.createFeatures(features, tokens, 3, null); + // The flag's entity type acts as gazetteer-like evidence for the name finder, and the region + // is decoded from the sequence with no dictionary. There is no sentiment feature: a record + // never fabricates a value it has no source for. + assertEquals(List.of("emoji.type=FLAG", "emoji.category=FLAGS", "emoji.region=DE"), features); + } + + @Test + void ordinaryAndDegenerateTokensContributeNothing() { + final AdaptiveFeatureGenerator generator = new EmojiAnnotationFeatureGenerator(); + final List<String> features = new ArrayList<>(); + final String[] tokens = {"word", "", ":-)", cp(0x1F1E9), cp(0x1F9F8)}; + for (int i = 0; i < tokens.length; i++) { + generator.createFeatures(features, tokens, i, null); + } + // Includes the lone regional indicator: bulk feature generation must survive damaged text. + assertTrue(features.isEmpty(), "Unexpected features: " + features); + } + + @Test + void joinedFactsFlowIntoFeaturesThroughAConfiguredAnnotator() { + final EmojiAnnotator annotator = new EmojiAnnotator((symbol, isoRegion) -> Map.of()); + final AdaptiveFeatureGenerator generator = new EmojiAnnotationFeatureGenerator(annotator); + final List<String> features = new ArrayList<>(); + generator.createFeatures(features, new String[] {cp(0x1F642)}, 0, null); + assertEquals(List.of("emoji.sentiment=1", "emoji.type=FACE", + "emoji.category=SMILEYS_AND_EMOTION"), features); + assertThrows(IllegalArgumentException.class, + () -> new EmojiAnnotationFeatureGenerator(null)); + } + + @Test + void wiredThroughAFeatureGenerationDescriptor() throws Exception { + // The opt-in seam: only a descriptor that names the factory activates the generator; no + // default descriptor is touched. + final String descriptor = "<featureGenerators name=\"test\">\n" + + " <generator class=\"opennlp.tools.util.featuregen.AggregatedFeatureGeneratorFactory\">\n" + + " <generator class=\"opennlp.tools.util.featuregen.TokenFeatureGeneratorFactory\"/>\n" + + " <generator class=\"" + + "opennlp.tools.util.featuregen.EmojiAnnotationFeatureGeneratorFactory\"/>\n" + + " </generator>\n" + + "</featureGenerators>\n"; + final AdaptiveFeatureGenerator generator = GeneratorFactory.create( + new ByteArrayInputStream(descriptor.getBytes(StandardCharsets.UTF_8)), null); + final List<String> features = new ArrayList<>(); + generator.createFeatures(features, new String[] {cp(0x1F622)}, 0, null); + assertTrue(features.contains("w=" + cp(0x1F622)), "Missing token feature in: " + features); + assertTrue(features.contains("emoji.sentiment=-2"), "Missing emoji feature in: " + features); + assertTrue(features.contains("emoji.type=FACE"), "Missing emoji feature in: " + features); + } + + @Test + void isStatelessUnderAdaptiveDataCalls() { + final AdaptiveFeatureGenerator generator = new EmojiAnnotationFeatureGenerator(); + generator.updateAdaptiveData(new String[] {"a"}, new String[] {"o"}); + generator.clearAdaptiveData(); + final List<String> features = new ArrayList<>(); + generator.createFeatures(features, new String[] {cp(0x1F642)}, 0, null); + assertEquals(3, features.size()); + } + + @Test + void featurePrefixesMatchTheAnnotationAttributes() { + // Guards the feature vocabulary: a rename here silently orphans trained models, so the names + // are pinned against the annotation attribute keys they project. + assertEquals("sentiment", EmojiAnnotation.SENTIMENT); + assertEquals("entityType", EmojiAnnotation.ENTITY_TYPE); + assertEquals("category", EmojiAnnotation.CATEGORY); + assertEquals("isoRegion", EmojiAnnotation.ISO_REGION); + } +} diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/EmojiAnnotationsTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/EmojiAnnotationsTest.java index 27c76d735..b7cf4962e 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/EmojiAnnotationsTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/EmojiAnnotationsTest.java @@ -135,6 +135,17 @@ public class EmojiAnnotationsTest { } } + @Test + void everyBundledSymbolStartsBeyondAscii() throws IOException { + // The feature generators fast-path tokens whose first char is ASCII without touching the + // annotation layer; this audit keeps that guard sound. It holds structurally: annotatable + // symbols are pictographs, and flags (regional indicators, the waving black flag) are + // supplementary-plane sequences handled by the derived layer. + for (final String symbol : bundled().keySet()) { + assertTrue(symbol.charAt(0) > 0x7F, "ASCII-leading annotated symbol: " + symbol); + } + } + // --- lookup behavior --- @Test diff --git a/opennlp-docs/src/docbkx/normalizer.xml b/opennlp-docs/src/docbkx/normalizer.xml index 8cbca7b9c..00e332f5c 100644 --- a/opennlp-docs/src/docbkx/normalizer.xml +++ b/opennlp-docs/src/docbkx/normalizer.xml @@ -568,6 +568,19 @@ for (Term term : analyzer.analyze(text)) { from the Unicode-licensed CLDR short names and <code>emoji-test.txt</code> group headers, and no third-party sentiment data set is included. </para> + <para> + The annotations are wired as strictly opt-in feature inputs to three components, each + through its existing extension seam so that default behavior and existing models are + unchanged: the name finder (and any consumer of feature generation descriptors) activates + <code>EmojiAnnotationFeatureGenerator</code> by naming + <code>EmojiAnnotationFeatureGeneratorFactory</code> in a descriptor, the document + categorizer takes <code>EmojiFeatureGenerator</code> alongside its defaults through the + <code>DoccatFactory</code> feature generator array, and sentiment analysis trains with + <code>EmojiSentimentFactory</code>, whose context generator adds the emoji evidence and is + re-created from the model manifest at prediction time. A flag token then carries + gazetteer-like evidence (<code>emoji.type=FLAG</code>, <code>emoji.region=DE</code>) with no + dictionary, and a heart or crying face contributes its coarse polarity directly. + </para> </section> <section xml:id="tools.normalizer.confusables">
