This is an automated email from the ASF dual-hosted git repository. krickert pushed a commit to branch feat/GH-1347-unicode-emoji-18 in repository https://gitbox.apache.org/repos/asf/opennlp.git
commit d5677642c081d364a9b8380b22d9a8f86a086658 Author: Kristian Rickert <[email protected]> AuthorDate: Thu Oct 1 18:35:15 2026 -0400 GH-1347: Annotate the 19 emoji added in Emoji 18.0 Add records to emoji-annotations.txt for cracking face, the leftwards and rightwards thumb signs with each of their skin tone sequences, monarch butterfly, pickle, lighthouse, meteor, eraser and net with handle, with the short names and group headers copied from the 18.0 emoji-test.txt. EmojiEntityType receives new constants for the thumb sign, sky and weather, tool and writing subgroups the table did not cover before; the NOTICE files cite emoji-test.txt 18.0 for the names. --- NOTICE | 2 +- .../src/main/appended-resources/META-INF/NOTICE | 2 +- .../tools/util/normalizer/EmojiEntityType.java | 11 ++- .../tools/util/normalizer/emoji-annotations.txt | 88 +++++++++++++++++++++- .../util/normalizer/EmojiAnnotationsTest.java | 10 ++- opennlp-distr/src/main/readme/NOTICE | 2 +- src/license/NOTICE.template | 2 +- 7 files changed, 108 insertions(+), 9 deletions(-) diff --git a/NOTICE b/NOTICE index 48b13bb6c..c89b80e45 100644 --- a/NOTICE +++ b/NOTICE @@ -133,7 +133,7 @@ opennlp-core/opennlp-runtime/src/main/resources/opennlp/tools/util/normalizer/em is licensed under the Apache License 2.0, but its name values (the CLDR short names) and its entity-type/category derivations are transcribed from the upstream emoji-test.txt (Emoji Keyboard/Display Test Data for UTS #51, -version 17.0), distributed under the Unicode License V3 reproduced in the +version 18.0), distributed under the Unicode License V3 reproduced in the LICENSE file. Its sentiment scores are original project judgments; no third-party sentiment data set is included. diff --git a/opennlp-core/opennlp-runtime/src/main/appended-resources/META-INF/NOTICE b/opennlp-core/opennlp-runtime/src/main/appended-resources/META-INF/NOTICE index 1d6e00212..e01bda94b 100644 --- a/opennlp-core/opennlp-runtime/src/main/appended-resources/META-INF/NOTICE +++ b/opennlp-core/opennlp-runtime/src/main/appended-resources/META-INF/NOTICE @@ -22,7 +22,7 @@ The project-authored annotation table opennlp/tools/util/normalizer/emoji-annotations.txt is licensed under the Apache License 2.0, but its name values (the CLDR short names) and its entity-type/category derivations are transcribed from the upstream -emoji-test.txt (version 17.0). +emoji-test.txt (version 18.0). Copyright (c) 1991-2026 Unicode, Inc. All rights reserved. Distributed under the Unicode License V3 reproduced in LICENSE. diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/EmojiEntityType.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/EmojiEntityType.java index 5c43c7840..5e271ad8d 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/EmojiEntityType.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/EmojiEntityType.java @@ -49,9 +49,18 @@ public enum EmojiEntityType { /** A vehicle, from the {@code transport-*} subgroups. */ VEHICLE, - /** A landmark or building, from the {@code place-*} subgroups. */ + /** A landmark or building, from the {@code place-*} subgroups and the lighthouse. */ LANDMARK, + /** A hand gesture, from the {@code hand-*} subgroups. */ + HAND, + + /** A sky or weather phenomenon, from the {@code sky & weather} subgroup. */ + SKY_AND_WEATHER, + + /** A hand tool or writing implement, from the {@code tool} and {@code writing} subgroups. */ + TOOL, + /** A flag; assigned by the derived layer from the code point sequence itself. */ FLAG } diff --git a/opennlp-core/opennlp-runtime/src/main/resources/opennlp/tools/util/normalizer/emoji-annotations.txt b/opennlp-core/opennlp-runtime/src/main/resources/opennlp/tools/util/normalizer/emoji-annotations.txt index ea416c164..5ed66be10 100644 --- a/opennlp-core/opennlp-runtime/src/main/resources/opennlp/tools/util/normalizer/emoji-annotations.txt +++ b/opennlp-core/opennlp-runtime/src/main/resources/opennlp/tools/util/normalizer/emoji-annotations.txt @@ -23,13 +23,14 @@ # # Attributes and their provenance: # name The CLDR short name as printed in the upstream emoji-test.txt (Emoji -# Keyboard/Display Test Data for UTS #51, version 17.0); source CLDR:annotation. +# Keyboard/Display Test Data for UTS #51, version 18.0); source CLDR:annotation. # sentiment A project-authored coarse polarity score, an integer in -2..2; source UNSPECIFIED, # the marker for a project judgment. No third-party sentiment data set is copied # here; in particular the Emoji Sentiment Ranking (CC BY-SA) is NOT used in any form. # entityType A coarse entity type (an EmojiEntityType constant), assigned by this project from # the subgroup header of the upstream emoji-test.txt; source UCD:emoji-test, the -# subgroup is cited in the notes. +# subgroup is cited in the notes. An entity type that departs from the subgroup is +# a project judgment, source UNSPECIFIED, with the subgroup still cited in the notes. # category A document-category hint (an EmojiCategory constant), the group header of the # upstream emoji-test.txt; source UCD:emoji-test, the group is cited in the notes. # @@ -38,7 +39,8 @@ # every record carries all four attributes. Flag emoji have no rows: their region is derived from # the code point sequence (see EmojiFlags), not bundled. # -# Code points, names, and group/subgroup headers checked against emoji-test.txt version 17.0. +# Code points, names, and group/subgroup headers checked against emoji-test.txt version 18.0; +# the notes column records which emoji-test.txt version a name was transcribed from. # # A line starting with '#' is a comment; data rows carry no inline comments. The notes column is # the fifth and final field, so it may contain ';'. @@ -172,6 +174,60 @@ 1F494 ; sentiment ; -2 ; UNSPECIFIED ; project judgment 1F494 ; entityType ; HEART ; UCD:emoji-test ; subgroup heart 1F494 ; category ; SMILEYS_AND_EMOTION ; UCD:emoji-test ; group Smileys & Emotion +1FAEB ; name ; cracking face ; CLDR:annotation ; emoji-test.txt v18.0 short name +1FAEB ; sentiment ; -1 ; UNSPECIFIED ; project judgment +1FAEB ; entityType ; FACE ; UCD:emoji-test ; subgroup face-smiling +1FAEB ; category ; SMILEYS_AND_EMOTION ; UCD:emoji-test ; group Smileys & Emotion +# +# People & Body +1FAF9 ; name ; leftwards thumb sign ; CLDR:annotation ; emoji-test.txt v18.0 short name +1FAF9 ; sentiment ; 0 ; UNSPECIFIED ; project judgment +1FAF9 ; entityType ; HAND ; UCD:emoji-test ; subgroup hand-fingers-closed +1FAF9 ; category ; PEOPLE_AND_BODY ; UCD:emoji-test ; group People & Body +1FAF9 1F3FB ; name ; leftwards thumb sign: light skin tone ; CLDR:annotation ; emoji-test.txt v18.0 short name +1FAF9 1F3FB ; sentiment ; 0 ; UNSPECIFIED ; project judgment +1FAF9 1F3FB ; entityType ; HAND ; UCD:emoji-test ; subgroup hand-fingers-closed +1FAF9 1F3FB ; category ; PEOPLE_AND_BODY ; UCD:emoji-test ; group People & Body +1FAF9 1F3FC ; name ; leftwards thumb sign: medium-light skin tone ; CLDR:annotation ; emoji-test.txt v18.0 short name +1FAF9 1F3FC ; sentiment ; 0 ; UNSPECIFIED ; project judgment +1FAF9 1F3FC ; entityType ; HAND ; UCD:emoji-test ; subgroup hand-fingers-closed +1FAF9 1F3FC ; category ; PEOPLE_AND_BODY ; UCD:emoji-test ; group People & Body +1FAF9 1F3FD ; name ; leftwards thumb sign: medium skin tone ; CLDR:annotation ; emoji-test.txt v18.0 short name +1FAF9 1F3FD ; sentiment ; 0 ; UNSPECIFIED ; project judgment +1FAF9 1F3FD ; entityType ; HAND ; UCD:emoji-test ; subgroup hand-fingers-closed +1FAF9 1F3FD ; category ; PEOPLE_AND_BODY ; UCD:emoji-test ; group People & Body +1FAF9 1F3FE ; name ; leftwards thumb sign: medium-dark skin tone ; CLDR:annotation ; emoji-test.txt v18.0 short name +1FAF9 1F3FE ; sentiment ; 0 ; UNSPECIFIED ; project judgment +1FAF9 1F3FE ; entityType ; HAND ; UCD:emoji-test ; subgroup hand-fingers-closed +1FAF9 1F3FE ; category ; PEOPLE_AND_BODY ; UCD:emoji-test ; group People & Body +1FAF9 1F3FF ; name ; leftwards thumb sign: dark skin tone ; CLDR:annotation ; emoji-test.txt v18.0 short name +1FAF9 1F3FF ; sentiment ; 0 ; UNSPECIFIED ; project judgment +1FAF9 1F3FF ; entityType ; HAND ; UCD:emoji-test ; subgroup hand-fingers-closed +1FAF9 1F3FF ; category ; PEOPLE_AND_BODY ; UCD:emoji-test ; group People & Body +1FAFA ; name ; rightwards thumb sign ; CLDR:annotation ; emoji-test.txt v18.0 short name +1FAFA ; sentiment ; 0 ; UNSPECIFIED ; project judgment +1FAFA ; entityType ; HAND ; UCD:emoji-test ; subgroup hand-fingers-closed +1FAFA ; category ; PEOPLE_AND_BODY ; UCD:emoji-test ; group People & Body +1FAFA 1F3FB ; name ; rightwards thumb sign: light skin tone ; CLDR:annotation ; emoji-test.txt v18.0 short name +1FAFA 1F3FB ; sentiment ; 0 ; UNSPECIFIED ; project judgment +1FAFA 1F3FB ; entityType ; HAND ; UCD:emoji-test ; subgroup hand-fingers-closed +1FAFA 1F3FB ; category ; PEOPLE_AND_BODY ; UCD:emoji-test ; group People & Body +1FAFA 1F3FC ; name ; rightwards thumb sign: medium-light skin tone ; CLDR:annotation ; emoji-test.txt v18.0 short name +1FAFA 1F3FC ; sentiment ; 0 ; UNSPECIFIED ; project judgment +1FAFA 1F3FC ; entityType ; HAND ; UCD:emoji-test ; subgroup hand-fingers-closed +1FAFA 1F3FC ; category ; PEOPLE_AND_BODY ; UCD:emoji-test ; group People & Body +1FAFA 1F3FD ; name ; rightwards thumb sign: medium skin tone ; CLDR:annotation ; emoji-test.txt v18.0 short name +1FAFA 1F3FD ; sentiment ; 0 ; UNSPECIFIED ; project judgment +1FAFA 1F3FD ; entityType ; HAND ; UCD:emoji-test ; subgroup hand-fingers-closed +1FAFA 1F3FD ; category ; PEOPLE_AND_BODY ; UCD:emoji-test ; group People & Body +1FAFA 1F3FE ; name ; rightwards thumb sign: medium-dark skin tone ; CLDR:annotation ; emoji-test.txt v18.0 short name +1FAFA 1F3FE ; sentiment ; 0 ; UNSPECIFIED ; project judgment +1FAFA 1F3FE ; entityType ; HAND ; UCD:emoji-test ; subgroup hand-fingers-closed +1FAFA 1F3FE ; category ; PEOPLE_AND_BODY ; UCD:emoji-test ; group People & Body +1FAFA 1F3FF ; name ; rightwards thumb sign: dark skin tone ; CLDR:annotation ; emoji-test.txt v18.0 short name +1FAFA 1F3FF ; sentiment ; 0 ; UNSPECIFIED ; project judgment +1FAFA 1F3FF ; entityType ; HAND ; UCD:emoji-test ; subgroup hand-fingers-closed +1FAFA 1F3FF ; category ; PEOPLE_AND_BODY ; UCD:emoji-test ; group People & Body # # Animals & Nature 1F436 ; name ; dog face ; CLDR:annotation ; emoji-test.txt v17.0 short name @@ -182,6 +238,10 @@ 1F431 ; sentiment ; 0 ; UNSPECIFIED ; project judgment 1F431 ; entityType ; ANIMAL ; UCD:emoji-test ; subgroup animal-mammal 1F431 ; category ; ANIMALS_AND_NATURE ; UCD:emoji-test ; group Animals & Nature +1FACC ; name ; monarch butterfly ; CLDR:annotation ; emoji-test.txt v18.0 short name +1FACC ; sentiment ; 0 ; UNSPECIFIED ; project judgment +1FACC ; entityType ; ANIMAL ; UCD:emoji-test ; subgroup animal-bug +1FACC ; category ; ANIMALS_AND_NATURE ; UCD:emoji-test ; group Animals & Nature # # Food & Drink 1F355 ; name ; pizza ; CLDR:annotation ; emoji-test.txt v17.0 short name @@ -192,6 +252,10 @@ 1F34E ; sentiment ; 0 ; UNSPECIFIED ; project judgment 1F34E ; entityType ; FOOD ; UCD:emoji-test ; subgroup food-fruit 1F34E ; category ; FOOD_AND_DRINK ; UCD:emoji-test ; group Food & Drink +1FADD ; name ; pickle ; CLDR:annotation ; emoji-test.txt v18.0 short name +1FADD ; sentiment ; 0 ; UNSPECIFIED ; project judgment +1FADD ; entityType ; FOOD ; UCD:emoji-test ; subgroup food-vegetable +1FADD ; category ; FOOD_AND_DRINK ; UCD:emoji-test ; group Food & Drink # # Travel & Places 1F697 ; name ; automobile ; CLDR:annotation ; emoji-test.txt v17.0 short name @@ -210,3 +274,21 @@ 1F5FC ; sentiment ; 0 ; UNSPECIFIED ; project judgment 1F5FC ; entityType ; LANDMARK ; UCD:emoji-test ; subgroup place-building 1F5FC ; category ; TRAVEL_AND_PLACES ; UCD:emoji-test ; group Travel & Places +1F6D9 ; name ; lighthouse ; CLDR:annotation ; emoji-test.txt v18.0 short name +1F6D9 ; sentiment ; 0 ; UNSPECIFIED ; project judgment +1F6D9 ; entityType ; LANDMARK ; UNSPECIFIED ; project judgment, UCD subgroup transport-water +1F6D9 ; category ; TRAVEL_AND_PLACES ; UCD:emoji-test ; group Travel & Places +1FA8B ; name ; meteor ; CLDR:annotation ; emoji-test.txt v18.0 short name +1FA8B ; sentiment ; 0 ; UNSPECIFIED ; project judgment +1FA8B ; entityType ; SKY_AND_WEATHER ; UCD:emoji-test ; subgroup sky & weather +1FA8B ; category ; TRAVEL_AND_PLACES ; UCD:emoji-test ; group Travel & Places +# +# Objects +1FA8C ; name ; eraser ; CLDR:annotation ; emoji-test.txt v18.0 short name +1FA8C ; sentiment ; 0 ; UNSPECIFIED ; project judgment +1FA8C ; entityType ; TOOL ; UCD:emoji-test ; subgroup writing +1FA8C ; category ; OBJECTS ; UCD:emoji-test ; group Objects +1FA8D ; name ; net with handle ; CLDR:annotation ; emoji-test.txt v18.0 short name +1FA8D ; sentiment ; 0 ; UNSPECIFIED ; project judgment +1FA8D ; entityType ; TOOL ; UCD:emoji-test ; subgroup tool +1FA8D ; category ; OBJECTS ; UCD:emoji-test ; group Objects diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/EmojiAnnotationsTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/EmojiAnnotationsTest.java index 918e2cb10..d1fbd1ccf 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/EmojiAnnotationsTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/EmojiAnnotationsTest.java @@ -70,7 +70,9 @@ public class EmojiAnnotationsTest { // The licensing audit: names come from the CLDR short names, entity type and category from // the emoji-test.txt group/subgroup headers, and sentiment is ALWAYS a project judgment // (UNSPECIFIED). A sentiment row with any other source would mean third-party sentiment data - // (for example the CC BY-SA Emoji Sentiment Ranking) leaked into the bundled file. + // (for example the CC BY-SA Emoji Sentiment Ranking) leaked into the bundled file. An entity + // type that departs from the subgroup is a project judgment too, and its notes must still + // cite the UCD subgroup it departs from. final Map<String, String> expectedSource = Map.of( EmojiAnnotation.NAME, "CLDR:annotation", EmojiAnnotation.SENTIMENT, "UNSPECIFIED", @@ -80,6 +82,12 @@ public class EmojiAnnotationsTest { for (final Map.Entry<String, EmojiAnnotation.Value> entry : annotation.attributes().entrySet()) { final EmojiAnnotation.Value value = entry.getValue(); + if (EmojiAnnotation.ENTITY_TYPE.equals(entry.getKey()) + && "UNSPECIFIED".equals(value.source())) { + assertTrue(value.notes().contains("UCD subgroup "), + "Project-judged entity type without its UCD subgroup: " + annotation.symbol()); + continue; + } assertEquals(expectedSource.get(entry.getKey()), value.source(), "Unexpected source for " + annotation.symbol() + " " + entry.getKey()); assertFalse(value.notes().isEmpty(), diff --git a/opennlp-distr/src/main/readme/NOTICE b/opennlp-distr/src/main/readme/NOTICE index 48b13bb6c..c89b80e45 100644 --- a/opennlp-distr/src/main/readme/NOTICE +++ b/opennlp-distr/src/main/readme/NOTICE @@ -133,7 +133,7 @@ opennlp-core/opennlp-runtime/src/main/resources/opennlp/tools/util/normalizer/em is licensed under the Apache License 2.0, but its name values (the CLDR short names) and its entity-type/category derivations are transcribed from the upstream emoji-test.txt (Emoji Keyboard/Display Test Data for UTS #51, -version 17.0), distributed under the Unicode License V3 reproduced in the +version 18.0), distributed under the Unicode License V3 reproduced in the LICENSE file. Its sentiment scores are original project judgments; no third-party sentiment data set is included. diff --git a/src/license/NOTICE.template b/src/license/NOTICE.template index a6b7ab035..b32c80043 100644 --- a/src/license/NOTICE.template +++ b/src/license/NOTICE.template @@ -133,7 +133,7 @@ opennlp-core/opennlp-runtime/src/main/resources/opennlp/tools/util/normalizer/em is licensed under the Apache License 2.0, but its name values (the CLDR short names) and its entity-type/category derivations are transcribed from the upstream emoji-test.txt (Emoji Keyboard/Display Test Data for UTS #51, -version 17.0), distributed under the Unicode License V3 reproduced in the +version 18.0), distributed under the Unicode License V3 reproduced in the LICENSE file. Its sentiment scores are original project judgments; no third-party sentiment data set is included.
