krickert commented on code in PR #1177: URL: https://github.com/apache/opennlp/pull/1177#discussion_r3621742298
########## opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/EmojiAnnotator.java: ########## @@ -0,0 +1,196 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.tools.util.normalizer; + +import java.util.Collection; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Optional; + +/** + * Assembles one {@link EmojiAnnotation} per symbol from bundled facts + * ({@link EmojiAnnotations}), derived facts ({@link EmojiFlags}), and optional joined facts + * ({@link EmojiAnnotationJoin}). {@link #annotate(Term)} keys on {@link Term#original()}; + * non-emoji tokens and degenerate flag-shaped text return empty. Instances are immutable and + * thread-safe when the join is. + * + * <p>Annotations are per-symbol metadata, not text transforms; a parallel surface beside + * {@link Term} rather than {@link Dimension} constants. See OPENNLP-1870.</p> + */ +public final class EmojiAnnotator { + + /** Feature-name prefix for the coarse sentiment score, for example {@code emojiSentiment=2}. */ + private static final String FEATURE_SENTIMENT_PREFIX = "emojiSentiment="; + + /** Feature-name prefix for the coarse entity type, for example {@code emojiType=HEART}. */ + private static final String FEATURE_TYPE_PREFIX = "emojiType="; + + /** + * Feature-name prefix for the document-category hint, for example + * {@code emojiCategory=SMILEYS_AND_EMOTION}. + */ + private static final String FEATURE_CATEGORY_PREFIX = "emojiCategory="; + + /** Feature-name prefix for a flag's ISO 3166 region, for example {@code emojiRegion=DE}. */ + private static final String FEATURE_REGION_PREFIX = "emojiRegion="; + + // Provenance tags of the derived facts; the mechanisms are defined by UTS #51. + private static final String FLAG_SEQUENCE = "UTS51:flag-sequence"; + private static final String TAG_SEQUENCE = "UTS51:tag-sequence"; + private static final String DERIVED_NOTE = "decoded from the code point sequence"; + + // The first non-ASCII code unit; every annotatable symbol starts at or beyond it. + private static final char FIRST_NON_ASCII = 0x80; + + private final EmojiAnnotationJoin join; + + /** + * Creates an annotator over the bundled and derived layers only. + */ + public EmojiAnnotator() { + this.join = null; + } + + /** + * Creates an annotator that also resolves joined facts through {@code join}. + * + * @param join The joined-facts hook, called while a record is assembled. Must not be + * {@code null}. + * @throws IllegalArgumentException if {@code join} is {@code null}. + */ + public EmojiAnnotator(EmojiAnnotationJoin join) { + if (join == null) { + throw new IllegalArgumentException("Join must not be null"); + } + this.join = join; + } + + /** + * Annotates one term, keyed on its {@link Term#original() original} text (see the class note on + * why the original layer is the one annotations describe). + * + * @param term The term to annotate. Must not be {@code null}. + * @return The assembled record, or empty when the term is not an annotated symbol. + * @throws IllegalArgumentException if {@code term} is {@code null}. + * @throws IllegalStateException if the configured join violates its contract (returns + * {@code null} or a key colliding with an existing attribute). + */ + public Optional<EmojiAnnotation> annotate(Term term) { + if (term == null) { + throw new IllegalArgumentException("Term must not be null"); + } + return annotate(term.original()); + } + + /** + * Annotates one symbol. + * + * @param symbol The code point sequence of one symbol (one token). U+FE0F presentation + * selectors are ignored. Must not be {@code null}. + * @return The assembled record, or empty when {@code symbol} is not an annotated symbol. + * @throws IllegalArgumentException if {@code symbol} is {@code null}. + * @throws IllegalStateException if the configured join violates its contract (returns + * {@code null} or a key colliding with an existing attribute). + */ + public Optional<EmojiAnnotation> annotate(CharSequence symbol) { + if (symbol == null) { + throw new IllegalArgumentException("Symbol must not be null"); + } + final EmojiAnnotation bundled = EmojiAnnotations.lookup(symbol).orElse(null); Review Comment: fixed per suggestion ########## opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/EmojiAnnotations.java: ########## @@ -0,0 +1,244 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.tools.util.normalizer; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.UncheckedIOException; +import java.nio.charset.StandardCharsets; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; + +/** + * The bundled-facts layer of the emoji annotation record store: license-clean, provenance-tagged + * attributes intrinsic to a pictograph (name, coarse sentiment, entity type, document category), + * loaded once from the project-authored {@code emoji-annotations.txt} resource. Each row of that + * file is one attribute of one symbol ({@code codepoints ; attribute ; value ; source ; notes}), + * so every value carries its own provenance and adding an attribute later is new rows plus loader + * support instead of a file-format break. The loader fails loud on an unknown attribute or a + * malformed row: the data and the code move together, so an unrecognized attribute is corruption, + * not extensibility. + * + * <p>Data licensing: the name values are the CLDR short names and the entity-type/category values + * are derived from the group and subgroup headers of the upstream {@code emoji-test.txt} + * (UTS #51, Unicode License V3, see the NOTICE file); the sentiment scores are original + * project judgments tagged {@code UNSPECIFIED}. No third-party sentiment data set is copied; in + * particular the Emoji Sentiment Ranking (CC BY-SA) is not used in any form.</p> + * + * <p>{@link EmojiAnnotator} is the accessor. Lookups strip U+FE0F (emoji presentation selector) + * because it does not change identity; bundled rows are keyed without it. Flag emoji have no + * bundled rows: region is derived, and gazetteer ids are never baked in.</p> + */ +public final class EmojiAnnotations { + + private static final String RESOURCE = "emoji-annotations.txt"; + + /** Starts a comment line in {@code emoji-annotations.txt}. */ + private static final String COMMENT_PREFIX = "#"; + + /** + * Field separator in {@code emoji-annotations.txt} + * ({@code codepoints ; attribute ; value ; source ; notes}). + */ + private static final String FIELD_SEPARATOR = ";"; + + /** + * Separates hex code points inside the code point field. The bundled table format uses ASCII + * space ({@code U+0020}), not a general whitespace class. + */ + private static final String MAPPING_CODE_POINT_SEPARATOR = " "; + + // The records keyed by code point sequence, loaded once when this class initializes. + private static final Map<String, EmojiAnnotation> ANNOTATIONS = load(); + + private EmojiAnnotations() { + } + + /** + * Returns the bundled annotation record of one emoji. + * + * @param symbol The code point sequence of one symbol, for example one {@link Term#original()} + * token. U+FE0F presentation selectors are ignored. Must not be {@code null}. + * @return The record, or empty when the bundled data does not annotate the symbol. + * @throws IllegalArgumentException if {@code symbol} is {@code null}. + * @throws IllegalStateException if the bundled data resource is missing. + * @throws UncheckedIOException if the bundled data resource cannot be read. + */ + public static Optional<EmojiAnnotation> lookup(CharSequence symbol) { + if (symbol == null) { + throw new IllegalArgumentException("Symbol must not be null"); + } + final String key = stripPresentationSelector(symbol); + if (key.isEmpty()) { + return Optional.empty(); + } + return Optional.ofNullable(ANNOTATIONS.get(key)); + } + + // Removes every U+FE0F VARIATION SELECTOR-16; allocation-free when none is present. + // Package-visible so EmojiAnnotator keys derived-only records the same way. + static String stripPresentationSelector(CharSequence symbol) { + final int length = symbol.length(); + int i = 0; + while (i < length && symbol.charAt(i) != 0xFE0F) { + i++; + } + if (i == length) { + return symbol.toString(); + } + final StringBuilder stripped = new StringBuilder(length - 1); + stripped.append(symbol, 0, i); + for (int k = i + 1; k < length; k++) { + final char c = symbol.charAt(k); + if (c != 0xFE0F) { + stripped.append(c); + } + } + return stripped.toString(); + } + + private static Map<String, EmojiAnnotation> load() { + try (InputStream in = EmojiAnnotations.class.getResourceAsStream(RESOURCE)) { + if (in == null) { + throw new IllegalStateException("Missing emoji annotation data resource: " + RESOURCE); + } + return parse(in); + } catch (IOException e) { + throw new UncheckedIOException("Unable to read emoji annotation data resource " + RESOURCE, e); + } + } + + // Package-private so the malformed-data handling can be exercised without the bundled resource. + // Parses rows of "codepoints ; attribute ; value ; source ; notes" with space-separated + // hexadecimal code points; '#' starts a comment line. The notes column is the fifth and final + // field, so it may contain ';'. + static Map<String, EmojiAnnotation> parse(InputStream in) throws IOException { Review Comment: done -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
