krickert commented on code in PR #1177: URL: https://github.com/apache/opennlp/pull/1177#discussion_r3621745680
########## opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/EmojiAnnotations.java: ########## @@ -0,0 +1,244 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.tools.util.normalizer; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.UncheckedIOException; +import java.nio.charset.StandardCharsets; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; + +/** + * The bundled-facts layer of the emoji annotation record store: license-clean, provenance-tagged + * attributes intrinsic to a pictograph (name, coarse sentiment, entity type, document category), + * loaded once from the project-authored {@code emoji-annotations.txt} resource. Each row of that + * file is one attribute of one symbol ({@code codepoints ; attribute ; value ; source ; notes}), + * so every value carries its own provenance and adding an attribute later is new rows plus loader + * support instead of a file-format break. The loader fails loud on an unknown attribute or a + * malformed row: the data and the code move together, so an unrecognized attribute is corruption, + * not extensibility. + * + * <p>Data licensing: the name values are the CLDR short names and the entity-type/category values + * are derived from the group and subgroup headers of the upstream {@code emoji-test.txt} + * (UTS #51, Unicode License V3, see the NOTICE file); the sentiment scores are original + * project judgments tagged {@code UNSPECIFIED}. No third-party sentiment data set is copied; in + * particular the Emoji Sentiment Ranking (CC BY-SA) is not used in any form.</p> + * + * <p>{@link EmojiAnnotator} is the accessor. Lookups strip U+FE0F (emoji presentation selector) + * because it does not change identity; bundled rows are keyed without it. Flag emoji have no + * bundled rows: region is derived, and gazetteer ids are never baked in.</p> + */ +public final class EmojiAnnotations { + + private static final String RESOURCE = "emoji-annotations.txt"; + + /** Starts a comment line in {@code emoji-annotations.txt}. */ + private static final String COMMENT_PREFIX = "#"; + + /** + * Field separator in {@code emoji-annotations.txt} + * ({@code codepoints ; attribute ; value ; source ; notes}). + */ + private static final String FIELD_SEPARATOR = ";"; + + /** + * Separates hex code points inside the code point field. The bundled table format uses ASCII + * space ({@code U+0020}), not a general whitespace class. + */ + private static final String MAPPING_CODE_POINT_SEPARATOR = " "; + + // The records keyed by code point sequence, loaded once when this class initializes. + private static final Map<String, EmojiAnnotation> ANNOTATIONS = load(); + + private EmojiAnnotations() { + } + + /** + * Returns the bundled annotation record of one emoji. + * + * @param symbol The code point sequence of one symbol, for example one {@link Term#original()} + * token. U+FE0F presentation selectors are ignored. Must not be {@code null}. + * @return The record, or empty when the bundled data does not annotate the symbol. + * @throws IllegalArgumentException if {@code symbol} is {@code null}. + * @throws IllegalStateException if the bundled data resource is missing. + * @throws UncheckedIOException if the bundled data resource cannot be read. + */ + public static Optional<EmojiAnnotation> lookup(CharSequence symbol) { + if (symbol == null) { + throw new IllegalArgumentException("Symbol must not be null"); + } + final String key = stripPresentationSelector(symbol); + if (key.isEmpty()) { + return Optional.empty(); + } + return Optional.ofNullable(ANNOTATIONS.get(key)); + } + + // Removes every U+FE0F VARIATION SELECTOR-16; allocation-free when none is present. + // Package-visible so EmojiAnnotator keys derived-only records the same way. + static String stripPresentationSelector(CharSequence symbol) { Review Comment: done -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
