This is an automated email from the ASF dual-hosted git repository. krickert pushed a commit to branch feat/GH-1347-unicode-emoji-18 in repository https://gitbox.apache.org/repos/asf/opennlp.git
commit a5df1d84f27885c6cc0c8337b54a8cf3aad803cf Author: Kristian Rickert <[email protected]> AuthorDate: Thu Oct 1 18:32:21 2026 -0400 GH-1347: Expect the Unicode Emoji 18.0 inventory in the normalizer tests The oracle now pins 4992 qualified sequences (3963 fully-qualified and 1029 minimally-qualified) and asserts that the 19 sequences new in Emoji 18.0 are removed. Against the bundled 17.0 data this fails with 21 failures: expected 4992 but was 4973 in the two count tests, and each 18.0 sequence is kept. --- .../EmojiCharSequenceNormalizerTest.java | 37 +++++++++++----------- .../normalizer/UnicodeEmojiSequencesParseTest.java | 2 +- 2 files changed, 20 insertions(+), 19 deletions(-) diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/EmojiCharSequenceNormalizerTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/EmojiCharSequenceNormalizerTest.java index 5017e267c..7c5d09c56 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/EmojiCharSequenceNormalizerTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/EmojiCharSequenceNormalizerTest.java @@ -42,10 +42,10 @@ public class EmojiCharSequenceNormalizerTest { EmojiCharSequenceNormalizer.getInstance(); /** - * The number of fully-qualified and minimally-qualified sequences in the Emoji 17.0 + * The number of fully-qualified and minimally-qualified sequences in the Emoji 18.0 * emoji-test.txt, the same number dev/UnicodeEmojiSequenceGenerator.java pins for that release. */ - private static final int EMOJI_17_SEQUENCE_COUNT = 4973; + private static final int EMOJI_18_SEQUENCE_COUNT = 4992; private static Stream<Arguments> emojiSequences() { return Stream.of( @@ -68,11 +68,11 @@ public class EmojiCharSequenceNormalizerTest { } /** - * The nine code points and ten modifier sequences that Emoji 18.0 added on top of 17.0. The - * bundled inventory is Emoji 17.0, the same version as the other bundled Unicode data, so - * these are not emoji to this normalizer yet. + * The nine code points and ten modifier sequences that Emoji 18.0 added on top of 17.0, the + * difference between the qualified lines of the two emoji-test.txt files. The bundled + * inventory is Emoji 18.0, so each of them is one complete emoji to this normalizer. */ - private static Stream<String> emoji18OnlySequences() { + private static Stream<String> emoji18AddedSequences() { return Stream.of(cp(0x1FAEB), cp(0x1FACC), cp(0x1FADD), cp(0x1F6D9), cp(0x1FA8B), cp(0x1FA8C), cp(0x1FA8D), cp(0x1FAF9), cp(0x1FAFA), cp(0x1FAF9, 0x1F3FB), cp(0x1FAF9, 0x1F3FC), cp(0x1FAF9, 0x1F3FD), @@ -82,9 +82,9 @@ public class EmojiCharSequenceNormalizerTest { } @ParameterizedTest - @MethodSource("emoji18OnlySequences") - void normalizeKeepsSequencesAddedAfterEmoji17(String sequence) { - Assertions.assertEquals("a" + sequence + "b", NORMALIZER.normalize("a" + sequence + "b")); + @MethodSource("emoji18AddedSequences") + void normalizeRemovesSequencesAddedInEmoji18(String sequence) { + Assertions.assertEquals("a b", NORMALIZER.normalize("a" + sequence + "b")); } @Test @@ -244,11 +244,12 @@ public class EmojiCharSequenceNormalizerTest { } /** - * Known Emoji 17.0 sequences, one or two per emoji-test.txt group, written down from the - * published list rather than read from the bundled file, so a wrong or truncated inventory - * fails here. The last seven are new in 17.0. + * Known emoji sequences up to Emoji 17.0, one or two per emoji-test.txt group, written down + * from the published list rather than read from the bundled file, so a wrong or truncated + * inventory fails here. The last seven are new in 17.0; {@link #emoji18AddedSequences()} + * covers the 18.0 additions. */ - private static Stream<Arguments> knownEmoji17Sequences() { + private static Stream<Arguments> knownEmojiSequences() { return Stream.of( Arguments.of(cp(0x1F600), "Smileys & Emotion: grinning face"), Arguments.of(cp(0x1F636, 0x200D, 0x1F32B, 0xFE0F), "Smileys & Emotion: face in clouds"), @@ -280,14 +281,14 @@ public class EmojiCharSequenceNormalizerTest { } @ParameterizedTest(name = "{1}") - @MethodSource("knownEmoji17Sequences") - void normalizeRemovesKnownEmoji17Sequences(String emoji, String description) { + @MethodSource("knownEmojiSequences") + void normalizeRemovesKnownEmojiSequences(String emoji, String description) { Assertions.assertEquals("a b", NORMALIZER.normalize("a" + emoji + "b")); } @Test - void bundledInventoryHoldsTheEmoji17SequenceCount() { - Assertions.assertEquals(EMOJI_17_SEQUENCE_COUNT, + void bundledInventoryHoldsTheEmoji18SequenceCount() { + Assertions.assertEquals(EMOJI_18_SEQUENCE_COUNT, UnicodeEmojiSequences.getInstance().sequenceCount()); } @@ -309,7 +310,7 @@ public class EmojiCharSequenceNormalizerTest { count++; } } - Assertions.assertEquals(EMOJI_17_SEQUENCE_COUNT, count); + Assertions.assertEquals(EMOJI_18_SEQUENCE_COUNT, count); } @Test diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/UnicodeEmojiSequencesParseTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/UnicodeEmojiSequencesParseTest.java index 274430de4..5c37300eb 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/UnicodeEmojiSequencesParseTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/UnicodeEmojiSequencesParseTest.java @@ -33,7 +33,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue; class UnicodeEmojiSequencesParseTest { private static final String WELL_FORMED = "# EmojiSequences.txt\n" - + "# Version: 17.0\n" + + "# Version: 18.0\n" + "S;1F600\n" + "S;1F44D 1F3FD\n" + "\n"
