This is an automated email from the ASF dual-hosted git repository. krickert pushed a commit to branch OPENNLP-1894-lattice-cjk in repository https://gitbox.apache.org/repos/asf/opennlp.git
commit 040ea88ddfa6cfb30cb86d735c09139875ad2772 Author: Kristian Rickert <[email protected]> AuthorDate: Thu Jul 16 01:32:44 2026 -0400 OPENNLP-1894: Keep lattice test sources ASCII-only via Unicode escapes, add an EUC-JP loading example --- .../tokenize/lattice/LatticeTokenizerTest.java | 68 ++++++++++--------- .../tokenize/lattice/LatticeUsageExampleTest.java | 77 +++++++++++++++++----- .../tokenize/lattice/UnigramSegmenterTest.java | 52 ++++++++------- 3 files changed, 126 insertions(+), 71 deletions(-) diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeTokenizerTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeTokenizerTest.java index b643786a9..83b823706 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeTokenizerTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeTokenizerTest.java @@ -33,6 +33,10 @@ import opennlp.tools.util.Span; /** * Tests the lattice segmenter against a project-authored miniature dictionary; no * external dictionary data is involved. + * + * <p>Source strings are written as Unicode escapes to keep this file ASCII-only; the + * class works over the same miniature Japanese dictionary as the sibling usage + * example, whose javadoc spells out each fixture word.</p> */ public class LatticeTokenizerTest { @@ -44,12 +48,12 @@ public class LatticeTokenizerTest { @BeforeAll static void loadDictionary() throws IOException { write("lexicon.csv", String.join("\n", - "東京,0,0,3000,noun,proper", - "京都,0,0,3000,noun,proper", - "東,0,0,6000,noun,common", - "都,0,0,4000,noun,suffix", - "に,0,0,1000,particle,case", - "行く,0,0,3000,verb,base", + "\u6771\u4EAC,0,0,3000,noun,proper", + "\u4EAC\u90FD,0,0,3000,noun,proper", + "\u6771,0,0,6000,noun,common", + "\u90FD,0,0,4000,noun,suffix", + "\u306B,0,0,1000,particle,case", + "\u884C\u304F,0,0,3000,verb,base", "")); write("matrix.def", "1 1\n0 0 0\n"); write("char.def", String.join("\n", @@ -79,9 +83,9 @@ public class LatticeTokenizerTest { @Test void testLatticePrefersTheCheaperSegmentation() { // the famous case: Tokyo+Metro must win over East+Kyoto - final String text = "東京都に行く"; + final String text = "\u6771\u4EAC\u90FD\u306B\u884C\u304F"; Assertions.assertArrayEquals( - new String[] {"東京", "都", "に", "行く"}, + new String[] {"\u6771\u4EAC", "\u90FD", "\u306B", "\u884C\u304F"}, tokenizer.tokenize(text)); Assertions.assertArrayEquals(new Span[] { new Span(0, 2), new Span(2, 3), new Span(3, 4), new Span(4, 6)}, @@ -91,7 +95,7 @@ public class LatticeTokenizerTest { @Test void testMorphemesCarryDictionaryFeatures() { final List<Morpheme> morphemes = - tokenizer.analyze("東京都に行く"); + tokenizer.analyze("\u6771\u4EAC\u90FD\u306B\u884C\u304F"); Assertions.assertEquals(4, morphemes.size()); Assertions.assertEquals(List.of("noun", "proper"), morphemes.get(0).features()); Assertions.assertEquals(List.of("particle", "case"), morphemes.get(2).features()); @@ -100,7 +104,7 @@ public class LatticeTokenizerTest { @Test void testUnknownLatinRunGroupsIntoOneMorpheme() { - final List<Morpheme> morphemes = tokenizer.analyze("ABCに行く"); + final List<Morpheme> morphemes = tokenizer.analyze("ABC\u306B\u884C\u304F"); Assertions.assertEquals(3, morphemes.size()); Assertions.assertEquals("ABC", morphemes.get(0).surface()); Assertions.assertEquals(true, morphemes.get(0).unknown()); @@ -109,17 +113,17 @@ public class LatticeTokenizerTest { @Test void testUnknownKanjiPreferOneMorphemeOverTwo() { - final List<Morpheme> morphemes = tokenizer.analyze("峠道に行く"); + final List<Morpheme> morphemes = tokenizer.analyze("\u5CE0\u9053\u306B\u884C\u304F"); Assertions.assertEquals(3, morphemes.size()); - Assertions.assertEquals("峠道", morphemes.get(0).surface()); + Assertions.assertEquals("\u5CE0\u9053", morphemes.get(0).surface()); Assertions.assertEquals(true, morphemes.get(0).unknown()); } @Test void testWhitespaceSeparatesAndIsNeverAMorpheme() { - final String text = "東京 に 行く"; + final String text = "\u6771\u4EAC \u306B \u884C\u304F"; Assertions.assertArrayEquals( - new String[] {"東京", "に", "行く"}, + new String[] {"\u6771\u4EAC", "\u306B", "\u884C\u304F"}, tokenizer.tokenize(text)); Assertions.assertArrayEquals(new Span[] { new Span(0, 2), new Span(3, 4), new Span(5, 7)}, @@ -144,13 +148,13 @@ public class LatticeTokenizerTest { */ @Test void testSingleCharacterInput() { - Assertions.assertArrayEquals(new String[] {"に"}, tokenizer.tokenize("に")); - Assertions.assertArrayEquals(new Span[] {new Span(0, 1)}, tokenizer.tokenizePos("に")); - Assertions.assertFalse(tokenizer.analyze("に").get(0).unknown()); + Assertions.assertArrayEquals(new String[] {"\u306B"}, tokenizer.tokenize("\u306B")); + Assertions.assertArrayEquals(new Span[] {new Span(0, 1)}, tokenizer.tokenizePos("\u306B")); + Assertions.assertFalse(tokenizer.analyze("\u306B").get(0).unknown()); - final List<Morpheme> unknown = tokenizer.analyze("峠"); + final List<Morpheme> unknown = tokenizer.analyze("\u5CE0"); Assertions.assertEquals(1, unknown.size()); - Assertions.assertEquals("峠", unknown.get(0).surface()); + Assertions.assertEquals("\u5CE0", unknown.get(0).surface()); Assertions.assertEquals(new Span(0, 1), unknown.get(0).span()); Assertions.assertTrue(unknown.get(0).unknown()); } @@ -163,9 +167,9 @@ public class LatticeTokenizerTest { */ @Test void testEntirelyUnknownInputGroupsIntoOneDefaultMorpheme() { - final List<Morpheme> morphemes = tokenizer.analyze("①②③"); + final List<Morpheme> morphemes = tokenizer.analyze("\u2460\u2461\u2462"); Assertions.assertEquals(1, morphemes.size()); - Assertions.assertEquals("①②③", morphemes.get(0).surface()); + Assertions.assertEquals("\u2460\u2461\u2462", morphemes.get(0).surface()); Assertions.assertEquals(new Span(0, 3), morphemes.get(0).span()); Assertions.assertTrue(morphemes.get(0).unknown()); Assertions.assertEquals(List.of("symbol", "unknown"), morphemes.get(0).features()); @@ -178,9 +182,9 @@ public class LatticeTokenizerTest { */ @Test void testMixedKnownAndUnknownRuns() { - final String text = "東京①に行く"; + final String text = "\u6771\u4EAC\u2460\u306B\u884C\u304F"; Assertions.assertArrayEquals( - new String[] {"東京", "①", "に", "行く"}, + new String[] {"\u6771\u4EAC", "\u2460", "\u306B", "\u884C\u304F"}, tokenizer.tokenize(text)); Assertions.assertArrayEquals(new Span[] { new Span(0, 2), new Span(2, 3), new Span(3, 4), new Span(4, 6)}, @@ -197,9 +201,9 @@ public class LatticeTokenizerTest { */ @Test void testSpansStayOriginalAfterLeadingWhitespace() { - final String text = " 東京都に行く"; + final String text = " \u6771\u4EAC\u90FD\u306B\u884C\u304F"; Assertions.assertArrayEquals( - new String[] {"東京", "都", "に", "行く"}, + new String[] {"\u6771\u4EAC", "\u90FD", "\u306B", "\u884C\u304F"}, tokenizer.tokenize(text)); Assertions.assertArrayEquals(new Span[] { new Span(2, 4), new Span(4, 5), new Span(5, 6), new Span(6, 8)}, @@ -213,7 +217,7 @@ public class LatticeTokenizerTest { @Test void testShortLexiconRowFailsLoud(@TempDir Path broken) throws IOException { Files.write(broken.resolve("lexicon.csv"), - "東,0,0\n".getBytes(StandardCharsets.UTF_8)); + "\u6771,0,0\n".getBytes(StandardCharsets.UTF_8)); Assertions.assertThrows(IOException.class, () -> MecabDictionary.load(broken)); } @@ -224,7 +228,7 @@ public class LatticeTokenizerTest { @Test void testNonNumericLexiconCostFailsLoud(@TempDir Path broken) throws IOException { Files.write(broken.resolve("lexicon.csv"), - "東,0,0,abc,noun\n".getBytes(StandardCharsets.UTF_8)); + "\u6771,0,0,abc,noun\n".getBytes(StandardCharsets.UTF_8)); Assertions.assertThrows(IOException.class, () -> MecabDictionary.load(broken)); } @@ -235,7 +239,7 @@ public class LatticeTokenizerTest { @Test void testMalformedMatrixLineFailsLoud(@TempDir Path broken) throws IOException { Files.write(broken.resolve("lexicon.csv"), - "東,0,0,3000,noun\n".getBytes(StandardCharsets.UTF_8)); + "\u6771,0,0,3000,noun\n".getBytes(StandardCharsets.UTF_8)); Files.write(broken.resolve("matrix.def"), "1 1\n0 0\n".getBytes(StandardCharsets.UTF_8)); Assertions.assertThrows(IOException.class, () -> MecabDictionary.load(broken)); } @@ -248,7 +252,7 @@ public class LatticeTokenizerTest { void testCharDefMappingWithoutCategoryFailsLoud(@TempDir Path broken) throws IOException { Files.write(broken.resolve("lexicon.csv"), - "東,0,0,3000,noun\n".getBytes(StandardCharsets.UTF_8)); + "\u6771,0,0,3000,noun\n".getBytes(StandardCharsets.UTF_8)); Files.write(broken.resolve("matrix.def"), "1 1\n0 0 0\n".getBytes(StandardCharsets.UTF_8)); Files.write(broken.resolve("char.def"), "DEFAULT 0 1 0\n0x4E00..0x9FFF\n".getBytes(StandardCharsets.UTF_8)); @@ -265,7 +269,7 @@ public class LatticeTokenizerTest { void testMissingDefaultTemplateFailsLoudAtTokenizeTime(@TempDir Path partial) throws IOException { Files.write(partial.resolve("lexicon.csv"), - "東,0,0,3000,noun\n".getBytes(StandardCharsets.UTF_8)); + "\u6771,0,0,3000,noun\n".getBytes(StandardCharsets.UTF_8)); Files.write(partial.resolve("matrix.def"), "1 1\n0 0 0\n".getBytes(StandardCharsets.UTF_8)); Files.write(partial.resolve("char.def"), "DEFAULT 0 1 0\nKANJI 0 0 2\n0x4E00..0x9FFF KANJI\n" @@ -274,13 +278,13 @@ public class LatticeTokenizerTest { "KANJI,0,0,8000,noun\n".getBytes(StandardCharsets.UTF_8)); final LatticeTokenizer limited = new LatticeTokenizer(MecabDictionary.load(partial)); - Assertions.assertThrows(IllegalStateException.class, () -> limited.analyze("①")); + Assertions.assertThrows(IllegalStateException.class, () -> limited.analyze("\u2460")); } @Test void testMalformedDictionariesFailLoud(@TempDir Path broken) throws IOException { Files.write(broken.resolve("lexicon.csv"), - "東,0,0,3000,noun\n".getBytes(StandardCharsets.UTF_8)); + "\u6771,0,0,3000,noun\n".getBytes(StandardCharsets.UTF_8)); Assertions.assertThrows(IOException.class, () -> MecabDictionary.load(broken)); Files.write(broken.resolve("matrix.def"), "1 1\n0 0 0\n".getBytes(StandardCharsets.UTF_8)); diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeUsageExampleTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeUsageExampleTest.java index a5e2a39d1..a61969c90 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeUsageExampleTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeUsageExampleTest.java @@ -19,6 +19,7 @@ package opennlp.tools.tokenize.lattice; import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; @@ -39,6 +40,13 @@ import opennlp.tools.util.Span; * with a {@link UnigramSegmenter}. Everything is written to a temporary directory by * the test itself; no external dictionary or lexicon data and no network access are * involved. + * + * <p>Source strings are written as Unicode escapes to keep this file ASCII-only. The + * Japanese fixture words are Tokyo (U+6771 U+4EAC), Kyoto (U+4EAC U+90FD), east + * (U+6771), the metropolis suffix (U+90FD), the case particle ni (U+306B), and the + * verb iku, to go (U+884C U+304F); the Chinese fixture words are wo, I (U+6211), + * laidao, arrive (U+6765 U+5230), Beijing (U+5317 U+4EAC), and Tiananmen + * (U+5929 U+5B89 U+95E8).</p> */ public class LatticeUsageExampleTest { @@ -120,12 +128,12 @@ public class LatticeUsageExampleTest { // files every mecab-format distribution contains, wrapped like a release archive. final byte[] archive = gzippedTar(new String[][] { {"mini-dict-0.1/lexicon.csv", String.join("\n", - "東京,0,0,3000,noun,proper", - "京都,0,0,3000,noun,proper", - "東,0,0,6000,noun,common", - "都,0,0,4000,noun,suffix", - "に,0,0,1000,particle,case", - "行く,0,0,3000,verb,base", + "\u6771\u4EAC,0,0,3000,noun,proper", + "\u4EAC\u90FD,0,0,3000,noun,proper", + "\u6771,0,0,6000,noun,common", + "\u90FD,0,0,4000,noun,suffix", + "\u306B,0,0,1000,particle,case", + "\u884C\u304F,0,0,3000,verb,base", "")}, {"mini-dict-0.1/matrix.def", "1 1\n0 0 0\n"}, {"mini-dict-0.1/char.def", String.join("\n", @@ -154,9 +162,9 @@ public class LatticeUsageExampleTest { // Load and tokenize; both views must agree and stay in original coordinates. final LatticeTokenizer tokenizer = new LatticeTokenizer(MecabDictionary.load(dictionaryDirectory)); - final String text = "東京都に行く"; + final String text = "\u6771\u4EAC\u90FD\u306B\u884C\u304F"; Assertions.assertArrayEquals( - new String[] {"東京", "都", "に", "行く"}, + new String[] {"\u6771\u4EAC", "\u90FD", "\u306B", "\u884C\u304F"}, tokenizer.tokenize(text)); Assertions.assertArrayEquals(new Span[] { new Span(0, 2), new Span(2, 3), new Span(3, 4), new Span(4, 6)}, @@ -165,7 +173,7 @@ public class LatticeUsageExampleTest { // The analyze view adds the dictionary's feature columns to every morpheme. final List<Morpheme> morphemes = tokenizer.analyze(text); Assertions.assertEquals(4, morphemes.size()); - Assertions.assertEquals("東京", morphemes.get(0).surface()); + Assertions.assertEquals("\u6771\u4EAC", morphemes.get(0).surface()); Assertions.assertEquals(List.of("noun", "proper"), morphemes.get(0).features()); Assertions.assertFalse(morphemes.get(0).unknown()); } @@ -180,19 +188,58 @@ public class LatticeUsageExampleTest { // One word, its count, and an optional tag per line, whitespace separated. final Path lexicon = work.resolve("words.txt"); Files.write(lexicon, String.join("\n", - "我 5000 r", - "来到 2000 v", - "北京 3000 ns", - "天安门 1200 ns", + "\u6211 5000 r", + "\u6765\u5230 2000 v", + "\u5317\u4EAC 3000 ns", + "\u5929\u5B89\u95E8 1200 ns", "").getBytes(StandardCharsets.UTF_8)); final UnigramSegmenter segmenter = UnigramSegmenter.load(lexicon); - final String text = "我来到北京天安门"; + final String text = "\u6211\u6765\u5230\u5317\u4EAC\u5929\u5B89\u95E8"; Assertions.assertArrayEquals( - new String[] {"我", "来到", "北京", "天安门"}, + new String[] {"\u6211", "\u6765\u5230", "\u5317\u4EAC", "\u5929\u5B89\u95E8"}, segmenter.tokenize(text)); Assertions.assertArrayEquals(new Span[] { new Span(0, 1), new Span(1, 3), new Span(3, 5), new Span(5, 8)}, segmenter.tokenizePos(text)); } + + /** + * Walks the non-UTF-8 flow that widely used Japanese distributions require: the same + * miniature dictionary is written to disk encoded in EUC-JP and loaded through the + * charset-taking overload. The segmentation must match the UTF-8 run exactly, which + * shows the encoding is a property of loading, not of tokenization. + */ + @Test + void testLoadAnEucJpEncodedDictionary(@TempDir Path work) throws IOException { + final Charset eucJp = Charset.forName("EUC-JP"); + Files.write(work.resolve("lexicon.csv"), String.join("\n", + "\u6771\u4EAC,0,0,3000,noun,proper", + "\u4EAC\u90FD,0,0,3000,noun,proper", + "\u6771,0,0,6000,noun,common", + "\u90FD,0,0,4000,noun,suffix", + "\u306B,0,0,1000,particle,case", + "\u884C\u304F,0,0,3000,verb,base", + "").getBytes(eucJp)); + Files.write(work.resolve("matrix.def"), "1 1\n0 0 0\n".getBytes(eucJp)); + Files.write(work.resolve("char.def"), String.join("\n", + "DEFAULT 0 1 0", + "KANJI 0 0 2", + "HIRAGANA 0 1 0", + "", + "0x3041..0x3096 HIRAGANA", + "0x4E00..0x9FFF KANJI", + "").getBytes(eucJp)); + Files.write(work.resolve("unk.def"), String.join("\n", + "DEFAULT,0,0,10000,symbol,unknown", + "KANJI,0,0,8000,noun,unknown", + "HIRAGANA,0,0,9000,particle,unknown", + "").getBytes(eucJp)); + + final LatticeTokenizer tokenizer = + new LatticeTokenizer(MecabDictionary.load(work, eucJp)); + Assertions.assertArrayEquals( + new String[] {"\u6771\u4EAC", "\u90FD", "\u306B", "\u884C\u304F"}, + tokenizer.tokenize("\u6771\u4EAC\u90FD\u306B\u884C\u304F")); + } } diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/UnigramSegmenterTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/UnigramSegmenterTest.java index f086eafa2..9725b8c72 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/UnigramSegmenterTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/UnigramSegmenterTest.java @@ -30,18 +30,22 @@ import opennlp.tools.util.Span; /** * Tests the frequency-driven segmenter against a project-authored miniature lexicon; * no external lexicon data is involved. + * + * <p>Source strings are written as Unicode escapes to keep this file ASCII-only; the + * class works over the same miniature Chinese frequency lexicon as the sibling usage + * example, whose javadoc spells out each fixture word.</p> */ public class UnigramSegmenterTest { private static final String LEXICON = String.join("\n", - "我 5000 r", - "来到 2000 v", - "北京 3000 ns", - "清华大学 800 nt", - "清华 400 ns", - "华大 100 ns", - "大学 1500 n", - "的 9000 uj", + "\u6211 5000 r", + "\u6765\u5230 2000 v", + "\u5317\u4EAC 3000 ns", + "\u6E05\u534E\u5927\u5B66 800 nt", + "\u6E05\u534E 400 ns", + "\u534E\u5927 100 ns", + "\u5927\u5B66 1500 n", + "\u7684 9000 uj", ""); private static UnigramSegmenter segmenter; @@ -56,29 +60,29 @@ public class UnigramSegmenterTest { @Test void testPrefersWholeWordsOverFragments() { Assertions.assertArrayEquals( - new String[] {"我", "来到", "北京", "清华大学"}, - segmenter.tokenize("我来到北京清华大学")); + new String[] {"\u6211", "\u6765\u5230", "\u5317\u4EAC", "\u6E05\u534E\u5927\u5B66"}, + segmenter.tokenize("\u6211\u6765\u5230\u5317\u4EAC\u6E05\u534E\u5927\u5B66")); } @Test void testSpansStayInOriginalCoordinates() { Assertions.assertArrayEquals(new Span[] { new Span(0, 1), new Span(1, 3), new Span(3, 5), new Span(5, 9)}, - segmenter.tokenizePos("我来到北京清华大学")); + segmenter.tokenizePos("\u6211\u6765\u5230\u5317\u4EAC\u6E05\u534E\u5927\u5B66")); } @Test void testUnknownCharactersFallBackToSingles() { Assertions.assertArrayEquals( - new String[] {"我", "爱", "北京"}, - segmenter.tokenize("我爱北京")); + new String[] {"\u6211", "\u7231", "\u5317\u4EAC"}, + segmenter.tokenize("\u6211\u7231\u5317\u4EAC")); } @Test void testWhitespaceSeparates() { Assertions.assertArrayEquals( - new String[] {"北京", "大学"}, - segmenter.tokenize("北京 大学")); + new String[] {"\u5317\u4EAC", "\u5927\u5B66"}, + segmenter.tokenize("\u5317\u4EAC \u5927\u5B66")); Assertions.assertEquals(0, segmenter.tokenizePos(" ").length); } @@ -98,10 +102,10 @@ public class UnigramSegmenterTest { */ @Test void testSingleCharacterInput() { - Assertions.assertArrayEquals(new String[] {"我"}, segmenter.tokenize("我")); - Assertions.assertArrayEquals(new Span[] {new Span(0, 1)}, segmenter.tokenizePos("我")); - Assertions.assertArrayEquals(new String[] {"爱"}, segmenter.tokenize("爱")); - Assertions.assertArrayEquals(new Span[] {new Span(0, 1)}, segmenter.tokenizePos("爱")); + Assertions.assertArrayEquals(new String[] {"\u6211"}, segmenter.tokenize("\u6211")); + Assertions.assertArrayEquals(new Span[] {new Span(0, 1)}, segmenter.tokenizePos("\u6211")); + Assertions.assertArrayEquals(new String[] {"\u7231"}, segmenter.tokenize("\u7231")); + Assertions.assertArrayEquals(new Span[] {new Span(0, 1)}, segmenter.tokenizePos("\u7231")); } /** @@ -127,11 +131,11 @@ public class UnigramSegmenterTest { @Test void testMixedKnownAndUnknownRuns() { Assertions.assertArrayEquals( - new String[] {"我", "爱", "清华大学"}, - segmenter.tokenize("我爱清华大学")); + new String[] {"\u6211", "\u7231", "\u6E05\u534E\u5927\u5B66"}, + segmenter.tokenize("\u6211\u7231\u6E05\u534E\u5927\u5B66")); Assertions.assertArrayEquals(new Span[] { new Span(0, 1), new Span(1, 2), new Span(2, 6)}, - segmenter.tokenizePos("我爱清华大学")); + segmenter.tokenizePos("\u6211\u7231\u6E05\u534E\u5927\u5B66")); } /** @@ -140,9 +144,9 @@ public class UnigramSegmenterTest { */ @Test void testSpansStayOriginalAfterLeadingWhitespace() { - final String text = " 我来到北京"; + final String text = " \u6211\u6765\u5230\u5317\u4EAC"; Assertions.assertArrayEquals( - new String[] {"我", "来到", "北京"}, + new String[] {"\u6211", "\u6765\u5230", "\u5317\u4EAC"}, segmenter.tokenize(text)); Assertions.assertArrayEquals(new Span[] { new Span(2, 3), new Span(3, 5), new Span(5, 7)},
