This is an automated email from the ASF dual-hosted git repository.
krickert pushed a change to branch feat/GH-1347-unicode-emoji-18
in repository https://gitbox.apache.org/repos/asf/opennlp.git
from 318e632a7 Minor: Regenerated NOTICE File for
7b1dccfb6aecfad8f2c85da69e7463078356aef3 (#1344)
add e6ad7ee54 OPENNLP-1932: Regex removal (4/10): Match model finder
wildcards without regular expressions (#1278)
add 13d8ecf97 OPENNLP-1990: Regex removal (5c/10): Validate MASC
identifiers and anchors (#1322)
add b53d6a3b9 OPENNLP-1989: Regex removal (5b/10): Preserve feature
identities in event parsing (#1321)
add 33fd09c93 OPENNLP-1929: Regex Removal (7/10): BasicContextGenerator
takes the separator literally and splits on Unicode whitespace by default
(#1280)
add ac0281a98 Isolate ObjectInputFilter usage in BaseModel to allow model
loading on Android (#1346)
add ceb2bd880 OPENNLP-1974: Normalize complete Unicode emoji sequences
(#1313)
add 55d1dd0c0 OPENNLP-1975: Regex removal (9/10): Correct the output of
TokenSampleStream, ParserTool, ConlluStream, DownloadUtil and
FeatureGeneratorUtil (#1314)
add 83c1283df Minor: Regenerated NOTICE File for
55d1dd0c01bf502142eaf08abe2a337a7bed647f (#1350)
add 86bab9d57 OPENNLP-1975: Address follow-up review comments from #1314
(#1351)
add 773f107c4 Minor: Move StringUtil tests to opennlp-api, share test
fixtures, reject null in StringUtil.isEmpty and isBlank (#1361)
add 932dbd94c Minor: Share layer alignment and span mapping across
document annotators (#1357)
add 36cd43a8c Minor: Share ISO 639 code conversion via
LanguageCodeValidator (#1356)
add 20e63af11 Minor: Share one requireNonNullArg helper across modules
(#1353)
add 0e2676667 Minor: Fix StringUtil.isEmpty and isBlank null checks after
the ArgumentChecks move (#1362)
add a6c524390 Minor: Reuse StringUtil split, BOM, digit and
line-terminator helpers (#1352)
add e3bd9c49b Minor: Route limit lookup and entry-count checks through
ResourceLimits (#1355)
add be98633a9 Minor: Share one hex code point parser and comment stripper
in opennlp-api (#1354)
add e3d644c3c Minor: Reuse the runtime email scan in the spellcheck URL
guard (#1360)
add 1c240a26c Minor: Move the lattice tokenizer package to opennlp-runtime
(#1367)
add 5dbadc253 Minor: Move Wordpiece and chunker report tests next to their
code (#1363)
add f25fefa82 Minor: Keep OffsetAwareNormalizer and StringUtil Javadoc to
api contracts (#1364)
add 06a9a4946 Minor: Compose AlignedText stages with AlignedText.andThen
(#1358)
add c4c596c43 Minor: Mark AbstractDL chunking and normalization helpers
@Internal (#1368)
add 41c00ed9e Minor: Align the manual with the TermAnalyzer stemmer
thread-safety contract (#1366)
add 1f7d2604d Minor: Document RealValueFileEventStream.parseEvent as
public API (#1365)
add c70dc9196 Minor: Exclude bundled CaseFolding.txt from the RAT check
(#1369)
add a033e05f0 Minor: Share the jar scan of the Simple and Directory model
finders (#1374)
add 4962c2d04 Minor: Correct factually wrong Javadoc and manual statements
(#1375)
add ffc43e688 Minor: Share BERT tokens and softmax in opennlp-dl, load
vocab files with empty lines (#1359)
add 4c523a083 Minor: Make the RAT check read rat-excludes and fail on
unapproved files (#1386)
add 56e4eb27e Minor: Format outcome probabilities thread-safely in
AbstractModel (#1371)
add 942edcb99 Minor: Run SpellCorrectingTokenStream per token for any
normalizer (#1372)
add deaae403d Minor: Build the jar mask from getJarModelPrefix on each
model finder scan (#1387)
add e7b9f2e3e Minor: Reject a char.def range end without the 0x prefix
(#1370)
add 9ca5e0385 StringUtil.toUpperCase does not uppercase
supplementary-plane letters (#1381)
add 47369544c (fix) GH-1378: MeCab CSV reader accepts unterminated quoted
fields (#1383)
add a368de9cd (fix) GH-1379: Morpheme accepts a surface that does not
match its span (#1384)
add 197933cf9 OPENNLP-1934: Add explicit tokenizer character policies
(#1349)
add 42877392f Minor: Use ArgumentChecks for inline null checks in
normalizer and tokenize (#1400)
add fce5fdbd6 Minor: Use ArgumentChecks for the inline null checks in
opennlp-api (#1399)
add c8536d2c1 Minor: Use ArgumentChecks for inline null checks in ml,
model-resolver, spellcheck (#1402)
add 35e97920d Minor: Use ArgumentChecks for the remaining inline null
checks in runtime (#1401)
add d742b18f0 MorfologikTagDictionary is not thread-safe and fails on
entries without a tag (#1382)
add b2e843340 Bump org.apache.maven:apache-maven from 3.9.16 to 3.10.0
(#1405)
add dd45c0042 Bump com.puppycrawl.tools:checkstyle from 13.3.0 to 14.3.0
(#1407)
add a6871186b GH-1408: Rename ArgumentChecks to ParamChecks (#1409)
add 6c8fdd5c7 GH-1397: Reject null spellcheck arguments with
IllegalArgumentException (#1412)
add 169827669 GH-1396: Use ParamChecks for null arguments in
opennlp-runtime (#1411)
add 66b1719bd GH-1395: Reject null arguments in opennlp-api with
IllegalArgumentException (#1410)
add 19f201e80 GH-1414: Store mvnw.cmd with LF line endings (#1415)
add c98b76298 GH-1398: Add requireNonEmpty and requireNonNegative to
ParamChecks (#1413)
add 1fd51808e Spellcheck correction loses the capital of a
supplementary-plane initial (#1385)
add 8b8a4dc86 GH-1422: Clean up opennlp-morfologik builder and Javadoc
(#1430)
add af742d8bd GH-1422: Clean up NumberUtil and its tests in opennlp-uima
(#1432)
add e1a807c47 GH-1388: Load bundled Unicode data through one recoverable
loader (#1373)
add 42f446a5a GH-1422: Clean up the spellcheck normalizer and loader
(#1431)
add 95b1ac5ff GH-1422: Clean up the report listeners in opennlp-cli (#1429)
add d7eadeda2 GH-1422: Clean up the stemmer, Hunspell and NameFinderDL
evals (#1434)
add 0a363d573 GH-1421: Clean up the AD, MASC, CoNLL-U and Brat readers
(#1428)
add 97e4fb13f GH-1418: Clean up event parsing and context generators in
the ml modules (#1425)
add 0ca246230 GH-1417: Remove duplicated logic in opennlp-api and the
lattice tokenizer (#1424)
add ef04afe6d GH-1416: Correct and trim opennlp-api Javadoc (#1423)
add 9a6a927a5 GH-1422: Remove duplication in StringUtilTest and the
chunker listener test (#1433)
add 11a136cc6 GH-1441: Report malformed lattice dictionary content as
InvalidFormatException (#1443)
add bca31bb7c OPENNLP-1988: Load WordPiece vocabularies from
tokenizer.json (#1319)
add 013b789b1 GH-1394: Load downloaded models by language when the model
host cannot be reached (#1435)
add 5a6347ac2 GH-1436: Document thread-safety limits of
WhitespaceTokenizer.INSTANCE and the process-wide modes (#1437)
add cf81387cc GH-1438: Share code point validation and trim scans in
CharClass and CodePointSet (#1439)
add 617d38996 GH-1440: Isolate WordpieceTokenizer from the shared
WhitespaceTokenizer and share punctuation isolation (#1442)
add a9d226ce9 GH-1445: Run the CI build once per job and render
README.html from README.md only (#1446)
add e039a7fb7 GH-1393: Cache the downloaded test models between CI runs
(#1444)
add deb80bae8 OPENNLP-547: Add a transition-based dependency parser
component (#1236)
add 139b9ddb3 GH-1449: Share ONNX inference between DocumentCategorizerDL
and NameFinderDL with one failure message (#1450)
add ce2da5357 GH-1447: Reject self-closing AD markup tags as opening tags
(#1448)
add 779420863 GH-1451: Share staged install, checksum and tar helpers
across the dictionary installers (#1452)
add 8d2a8c310 GH-1419: Correct and trim opennlp-runtime Javadoc (#1426)
add eef3909dc GH-1420: Remove duplication and add readResolve in
opennlp-runtime (#1427)
add ef814bdba GH-1454: Tidy document, stemmer, corpora and doccat chapters
and README (#1455)
add 84a1d6268 GH-1460: Match escaped tags in the bundled Spanish head
rules (#1461)
add e286928a1 GH-1458: Parse name-finder tags and parse-tree strings
without regular expressions (#1459)
add a6f19f91e GH-1449: Align DocumentCategorizerDLEval with the shared
ONNX failure message (#1463)
add 46d9d96a1 GH-1456: Tighten normalizer, tokenizer, introduction and
language detector chapters (#1457)
add 7a560bd7a OPENNLP-1935: Regex removal (10/10): Forbid java.util.regex
in production code (#1348)
add 4f22baa93 GH-1464: Release the per-thread stemmer state of a
TermAnalyzer and reuse the matching analyzer (#1465)
add 050922919 GH-1466: Describe current behaviour instead of 3.0 changes
in the user manual (#1467)
add c31792e9d GH-1475: Say that opennlp-dl is not in the binary
distribution (#1477)
add 9306b4698 Bump actions/download-artifact from 8.0.1 to 8.0.2 (#1484)
add 6c51e3dbc Bump actions/upload-artifact from 7.0.1 to 7.0.2 (#1487)
add 6a0928ece Bump org.apache.commons:commons-secure-xml from 1.0.0 to
1.1.0 (#1490)
add 40fa1e47d Bump onnxruntime.version from 1.30.0 to 1.31.0 (#1491)
add 4713d8257 Bump io.github.classgraph:classgraph from 4.8.196 to 4.8.197
(#1492)
add 230f0a683 Minor: Regenerated NOTICE File for
4713d82577dd3e9e48cd9855ccd891c2739324a4 (#1493)
add 16f90fd68 OPENNLP-1991: Add a feedforward neural dependency parser
(#1323)
add 2cad1f29f GH-1389, GH-1390: Let SentenceVectorsDL run on a GPU and fix
the first CUDA request in a fresh JVM (#1462)
add c58b4da61 GH-1499: Update ClassGraph to 5.0.0 (#1500)
add 021b731d5 GH-1476: Let the parsers take a caller-supplied POSTagger
and Chunker (#1478)
add d603165a5 Bump actions/setup-java from 6.0.1 to 6.1.0 (#1503)
add 95e990636 GH-1391: Cap the token positions of one
SentenceVectorsDL.embedAll inference (#1468)
add 0e95db1ce GH-1506: Make AbstractClassPathModelFinder.findModels
thread-safe (#1507)
add 5807d75b0 GH-1469: Add command-line tools for the feedforward
dependency parser (#1473)
add cbd43d5ca Minor: Regenerated NOTICE File for
5807d75b0ba43a4ad7c685678f6b2287a2d53c4b (#1504)
add 95695633b Bump io.github.hakky54:logcaptor from 2.12.7 to 2.13.0
(#1510)
new a5df1d84f GH-1347: Expect the Unicode Emoji 18.0 inventory in the
normalizer tests
new bcec97f6f GH-1347: Update the bundled Unicode Emoji data to 18.0
new be96f4a29 GH-1347: Expect annotation records for the 19 emoji added in
Emoji 18.0
new d5677642c GH-1347: Annotate the 19 emoji added in Emoji 18.0
The 4 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.gitattributes | 17 +
.github/CONTRIBUTING.md | 14 +
.github/dependabot.yml | 6 +
.github/workflows/license.yml | 2 +-
.github/workflows/maven.yml | 35 +-
.github/workflows/publish-snapshots.yml | 2 +-
.mvn/wrapper/maven-wrapper.properties | 2 +-
LICENSE | 4 +-
NOTICE | 33 +-
README.md | 17 +-
dev/README-hunspell-dictionaries.md | 4 +-
dev/README-unicode-emoji-sequences.md | 52 +
dev/UnicodeEmojiSequenceGenerator.java | 296 ++
dev/forbidden-regex.txt | 30 +
opennlp-api/pom.xml | 17 +-
.../opennlp/tools/commons/SuppressForbidden.java | 51 +
.../java/opennlp/tools/depparse/DependencyArc.java | 65 +
.../DependencyEvaluationMonitor.java} | 8 +-
.../opennlp/tools/depparse/DependencyParser.java | 50 +
.../opennlp/tools/depparse/DependencySample.java | 145 +
.../opennlp/tools/depparse/DependencyTree.java | 270 ++
.../java/opennlp/tools/doccat/DocumentSample.java | 7 +-
.../java/opennlp/tools/document/Annotation.java | 5 +-
.../main/java/opennlp/tools/document/Document.java | 42 +-
.../opennlp/tools/document/DocumentAnalyzer.java | 6 +-
.../opennlp/tools/document/DocumentAnnotator.java | 5 +-
.../opennlp/tools/document/DocumentAnnotators.java | 110 +-
.../opennlp/tools/document/DocumentMerges.java | 71 +
.../opennlp/tools/document/ImmutableDocument.java | 41 +-
.../main/java/opennlp/tools/document/LayerKey.java | 5 +-
.../main/java/opennlp/tools/document/Layers.java | 24 +-
.../opennlp/tools/embeddings/TextEmbedder.java | 6 +-
.../java/opennlp/tools/langdetect/Language.java | 6 +-
.../opennlp/tools/langdetect/LanguageSample.java | 15 +-
.../main/java/opennlp/tools/ml/model/Event.java | 13 +-
.../opennlp/tools/models/ClassPathModelFinder.java | 2 +
.../java/opennlp/tools/namefind/NameSample.java | 56 +-
.../tools/namefind/OffsetMappingNameFinder.java | 13 +-
.../src/main/java/opennlp/tools/parser/Parse.java | 114 +-
.../opennlp/tools/sentiment/SentimentSample.java | 9 +-
.../java/opennlp/tools/stemmer/StemmerFactory.java | 4 +-
.../java/opennlp/tools/termvector/TermVector.java | 13 +-
.../opennlp/tools/tokenize/BertNormalization.java | 49 +-
.../java/opennlp/tools/tokenize/MappedText.java | 161 +
.../java/opennlp/tools/tokenize/SubwordPiece.java | 14 +-
.../opennlp/tools/tokenize/SubwordTokenizer.java | 5 +
.../java/opennlp/tools/tokenize/TokenSample.java | 19 +-
.../tools/tokenize/WhitespaceTokenizer.java | 8 +-
.../opennlp/tools/tokenize/WordpieceEncoder.java | 204 +-
.../opennlp/tools/tokenize/WordpieceTokenizer.java | 62 +-
.../java/opennlp/tools/util/CompatibilityMode.java | 24 +-
.../opennlp/tools/util/LanguageCodeValidator.java | 131 +-
.../java/opennlp/tools/util/ObjectStreamUtils.java | 12 +-
.../main/java/opennlp/tools/util/ParamChecks.java | 278 ++
.../opennlp/tools/util/PlainTextByLineStream.java | 7 +-
.../java/opennlp/tools/util/ResourceLimits.java | 55 +-
.../main/java/opennlp/tools/util/StringUtil.java | 195 +-
.../java/opennlp/tools/util/WhitespaceMode.java | 23 +-
.../opennlp/tools/util/ext/ExtensionLoader.java | 11 +-
.../java/opennlp/tools/util/ext/ProviderSpec.java | 26 +-
.../java/opennlp/tools/util/ext/Providers.java | 17 +-
.../opennlp/tools/util/normalizer/AlignedText.java | 43 +-
.../opennlp/tools/util/normalizer/Alignment.java | 20 +-
.../opennlp/tools/util/normalizer/CharClass.java | 422 +-
.../tools/util/normalizer/CodePointSet.java | 105 +-
.../tools/util/normalizer/HexCodePoints.java | 213 +
.../util/normalizer/OffsetAwareNormalizer.java | 14 +-
.../opennlp/tools/util/normalizer/UnicodeDash.java | 38 +-
.../tools/util/normalizer/UnicodeWhitespace.java | 50 +-
.../test/java/opennlp/tools/PinWhitespaceMode.java | 22 +-
.../java/opennlp/tools/PinWhitespaceModeTest.java | 62 +
.../java/opennlp/tools/ResetWhitespaceMode.java | 22 +-
.../opennlp/tools/ResetWhitespaceModeTest.java | 58 +
.../java/opennlp/tools/RestoreDefaultLocale.java | 77 +
.../opennlp/tools/RestoreDefaultLocaleTest.java | 68 +
.../opennlp/tools/WhitespaceModeExtension.java | 74 +
.../tools/depparse/DependencySampleTest.java | 262 +
.../opennlp/tools/depparse/DependencyTreeTest.java | 185 +
.../tools/document/DocumentAnnotatorsTest.java | 182 +-
.../tools/document/DocumentContractTest.java | 4 +
.../opennlp/tools/tokenize/MappedTextTest.java | 88 +
.../WordpieceEncoderReferenceSequencesTest.java | 0
.../tools/tokenize/WordpieceEncoderTest.java | 37 +-
.../opennlp/tools/util/CompatibilityModeTest.java | 4 +
.../java/opennlp/tools/util/ParamChecksTest.java | 284 ++
.../opennlp/tools/util/ResourceLimitsTest.java | 106 +-
.../opennlp/tools/util/StringUtilLocaleTest.java | 19 +-
.../tools/util/StringUtilSplitNonEmptyTest.java | 116 +
.../java/opennlp/tools/util/StringUtilTest.java | 302 +-
.../tools/util/StringUtilUpperCaseTest.java | 35 +-
.../opennlp/tools/util/WhitespaceModeTest.java | 4 +
.../tools/util/normalizer/AlignedTextTest.java | 95 +
.../tools/util/normalizer/AlignmentTest.java | 8 +
.../tools/util/normalizer/CharClassTest.java | 111 +-
.../tools/util/normalizer/CodePointSetTest.java | 58 +
.../tools/util/normalizer/HexCodePointsTest.java | 162 +
opennlp-core/opennlp-cli/pom.xml | 27 +
.../tools/cmdline/AbstractEvaluatorTool.java | 12 +
.../opennlp/tools/cmdline/AbstractTrainerTool.java | 7 +
.../src/main/java/opennlp/tools/cmdline/CLI.java | 16 +
.../tools/cmdline/DetailedFMeasureListener.java | 19 +
.../tools/cmdline/FineGrainedReportListener.java | 53 +-
.../java/opennlp/tools/cmdline/ModelLoader.java | 9 +
.../DependencyEvaluationErrorListener.java} | 36 +-
.../DependencyModelLoader.java} | 18 +-
.../DependencyParserCrossValidatorTool.java | 81 +
.../depparse/DependencyParserEvaluatorTool.java | 59 +
.../DependencyParserMETool.java} | 31 +-
.../depparse/DependencyParserToolSupport.java | 149 +
.../depparse/DependencyParserTrainerTool.java | 69 +
.../FeedforwardDependencyModelLoader.java} | 18 +-
.../FeedforwardDependencyParserEvaluatorTool.java | 77 +
.../depparse/FeedforwardDependencyParserTool.java | 84 +
.../FeedforwardDependencyParserTrainerTool.java | 125 +
.../opennlp/tools/cmdline/parser/ParserTool.java | 70 +-
.../opennlp/tools/AbstractModelLoaderTest.java | 4 +-
.../cmdline/FineGrainedReportListenerTest.java | 10 +-
.../ChunkerDetailedFMeasureListenerTest.java | 38 +-
.../cmdline}/chunker/DummyChunkSampleStream.java | 5 +-
.../tools/cmdline}/chunker/DummyChunker.java | 6 +-
.../depparse/DependencyParserToolsTest.java | 379 ++
.../depparse/DependencyToolTestSupport.java | 134 +
.../FeedforwardDependencyParserToolsTest.java | 581 +++
.../tools/cmdline/parser/ParserToolTest.java | 92 +
.../tools/cmdline}/chunker/detailedOutput.txt | 0
.../opennlp/tools/cmdline}/chunker/output.txt | 0
opennlp-core/opennlp-formats/pom.xml | 27 +
.../tools/cmdline/StreamFactoryRegistry.java | 2 +
.../java/opennlp/tools/formats/ad/ADMetadata.java | 29 +-
.../tools/formats/ad/ADNameSampleStream.java | 56 +-
.../tools/formats/ad/ADSentenceSampleStream.java | 8 +-
.../opennlp/tools/formats/ad/ADSentenceStream.java | 8 +-
.../tools/formats/brat/BratAnnotationStream.java | 7 +-
.../conllu/ConlluDependencySampleStream.java | 293 ++
.../ConlluDependencySampleStreamFactory.java | 109 +
.../conllu/ConlluLemmaSampleStreamFactory.java | 6 +-
.../conllu/ConlluPOSSampleStreamFactory.java | 6 +-
.../opennlp/tools/formats/conllu/ConlluStream.java | 115 +-
.../opennlp/tools/formats/conllu/ConlluTagset.java | 55 +-
.../tools/formats/conllu/ConlluWordLine.java | 2 +-
.../opennlp/tools/formats/masc/MascDocument.java | 10 +-
.../tools/formats/masc/MascIdentifiers.java | 149 +
.../tools/formats/masc/MascNamedEntityParser.java | 12 +-
.../tools/formats/masc/MascPennTagParser.java | 18 +-
.../tools/formats/masc/MascSentenceParser.java | 10 +-
.../java/opennlp/tools/formats/masc/MascWord.java | 2 +-
.../opennlp/tools/formats/masc/MascWordParser.java | 16 +-
.../tools/formats/ad/ADNameSampleStreamTest.java | 3 +-
.../formats/ad/ADSentenceSampleStreamTest.java | 5 +-
.../tools/formats/ad/ADSentenceStreamTest.java | 31 +-
.../conllu/ConlluDependencyParserUsageTest.java | 251 +
.../ConlluDependencySampleStreamFactoryTest.java | 113 +
.../conllu/ConlluDependencySampleStreamTest.java | 453 ++
.../tools/formats/conllu/ConlluStreamTest.java | 97 +
.../tools/formats/conllu/ConlluTagsetTest.java | 63 +
.../tools/formats/conllu/ConlluTestLines.java} | 21 +-
.../tools/formats/masc/MascDocumentTest.java | 82 +
.../tools/formats/masc/MascIdentifiersTest.java | 107 +
.../formats/masc/MascNamedEntityParserTest.java | 70 +
.../tools/formats/masc/MascParserTestUtil.java | 85 +
.../tools/formats/masc/MascPennTagParserTest.java | 92 +
.../tools/formats/masc/MascSentenceParserTest.java | 68 +
.../tools/formats/masc/MascWordParserTest.java | 87 +
opennlp-core/opennlp-ml/opennlp-dl-gpu/README.md | 17 +
opennlp-core/opennlp-ml/opennlp-dl-gpu/pom.xml | 34 +
opennlp-core/opennlp-ml/opennlp-dl/README.md | 44 +-
.../src/main/java/opennlp/dl/AbstractDL.java | 689 ++-
.../src/main/java/opennlp/dl/JsonScan.java | 158 +-
.../dl/doccat/DocumentCategorizerConfig.java | 5 +-
.../opennlp/dl/doccat/DocumentCategorizerDL.java | 186 +-
.../java/opennlp/dl/namefinder/NameFinderDL.java | 211 +-
.../dl/vectors/OnnxTextEmbedderProvider.java | 9 +-
.../java/opennlp/dl/vectors/SentenceVectorsDL.java | 119 +-
.../java/opennlp/dl/AbstractDLChunkingTest.java | 24 +-
.../opennlp/dl/AbstractDLColdStartCudaTest.java | 139 +
.../java/opennlp/dl/AbstractDLSoftmaxTest.java | 105 +
.../src/test/java/opennlp/dl/GpuRequestTest.java | 116 +
.../opennlp/dl/InferenceOptionsValidationTest.java | 22 +
.../src/test/java/opennlp/dl/JsonScanTest.java | 57 +
.../src/test/java/opennlp/dl/LoadVocabTest.java | 552 ++-
.../src/test/java/opennlp/dl/OnnxTestSupport.java | 83 +
.../dl/doccat/DocumentCategorizerDLTest.java | 21 +-
.../opennlp/dl/namefinder/NameFinderDLTest.java | 18 +-
.../vectors/SentenceVectorsDLBatchLimitTest.java | 240 +
.../dl/vectors/SentenceVectorsDLEmbedderTest.java | 91 +-
.../SentenceVectorsDLInferenceOptionsTest.java | 195 +
.../java/opennlp/dl/vectors/TinyVectorModels.java | 152 +
.../src/test/resources/opennlp/dl/tokenizer.json | 79 +
.../tools/ml/naivebayes/NaiveBayesModel.java | 2 +-
.../opennlp/tools/ml/AbstractMLModelWriter.java | 4 +-
.../src/main/java/opennlp/tools/ml/BeamSearch.java | 62 +-
.../tools/ml/model/AbstractDataIndexer.java | 4 +-
.../java/opennlp/tools/ml/model/AbstractModel.java | 16 +-
.../tools/ml/model/AbstractModelReader.java | 36 +-
.../java/opennlp/tools/ml/model/EventFields.java | 73 +
.../opennlp/tools/ml/model/FileEventStream.java | 17 +-
.../tools/ml/model/RealValueFileEventStream.java | 55 +-
.../tools/ml/model/SimpleEventStreamBuilder.java | 94 +-
.../opennlp/tools/ml/AbstractEventStreamTest.java | 8 +
.../opennlp/tools/ml/model/AbstractModelTest.java | 116 +
.../java/opennlp/tools/ml/model/EventTest.java | 20 +-
.../tools/ml/model/FileEventStreamTest.java | 29 +
.../ml/model/RealValueFileEventStreamTest.java | 254 +-
.../ml/model/SimpleEventStreamBuilderTest.java | 252 +
.../tools/ml/libsvm/doccat/SvmDoccatModel.java | 14 +-
opennlp-core/opennlp-ml/opennlp-ml-maxent/pom.xml | 8 +
.../tools/ml/maxent/BasicContextGenerator.java | 73 +-
.../opennlp/tools/ml/maxent/ContextGenerator.java | 2 +
.../tools/ml/maxent/RealBasicEventStream.java | 32 +-
.../tools/ml/maxent/BasicContextGeneratorTest.java | 251 +
.../tools/ml/maxent/RealBasicEventStreamTest.java | 50 +-
.../tools/ml/perceptron/PerceptronModel.java | 2 +-
opennlp-core/opennlp-model-resolver/pom.xml | 5 +
.../tools/models/AbstractClassPathModelFinder.java | 178 +-
.../tools/models/ClassPathModelProvider.java | 2 +
.../models/DefaultClassPathModelProvider.java | 3 +
.../java/opennlp/tools/models/WildcardMatcher.java | 76 +
.../models/classgraph/ClassgraphModelFinder.java | 7 +-
.../tools/models/dir/DirectoryModelFinder.java | 31 +-
.../models/simple/SimpleClassPathModelFinder.java | 58 +-
...bstractClassPathModelFinderConcurrencyTest.java | 212 +
.../models/AbstractClassPathModelFinderTest.java | 313 ++
.../opennlp/tools/models/ModelFinderPathTest.java | 173 +
.../java/opennlp/tools/models/UnescapedUrls.java | 56 +
.../opennlp/tools/models/WildcardMatcherTest.java | 234 +
.../tools/models/dir/DirectoryModelFinderTest.java | 130 +
.../simple/SimpleClassPathModelFinderTest.java | 78 +
opennlp-core/opennlp-runtime/BENCHMARKS.md | 29 +
opennlp-core/opennlp-runtime/pom.xml | 27 +
.../java/opennlp/tools/ml/BeamSearchBenchmark.java | 11 +-
.../src/main/appended-resources/META-INF/LICENSE | 50 +
.../src/main/appended-resources/META-INF/NOTICE | 28 +
.../opennlp/tools/chunker/ChunkerAnnotator.java | 32 +-
.../java/opennlp/tools/chunker/ChunkerModel.java | 4 +-
.../opennlp/tools/depparse/ArcStandardBeam.java | 149 +
.../opennlp/tools/depparse/ArcStandardOracle.java | 148 +
.../opennlp/tools/depparse/ArcStandardState.java | 330 ++
.../opennlp/tools/depparse/AttachmentScores.java | 114 +
.../opennlp/tools/depparse/ContributionCache.java | 175 +
.../tools/depparse/DependencyContextGenerator.java | 348 ++
.../tools/depparse/DependencyCrossValidator.java | 170 +
.../tools/depparse/DependencyEvaluator.java | 157 +
.../tools/depparse/DependencyEventStream.java | 126 +
.../opennlp/tools/depparse/DependencyModel.java | 125 +
.../opennlp/tools/depparse/DependencyParserME.java | 189 +
.../opennlp/tools/depparse/FeedforwardContext.java | 151 +
.../tools/depparse/FeedforwardDependencyModel.java | 186 +
.../depparse/FeedforwardDependencyParser.java | 203 +
.../depparse/FeedforwardDependencyTrainer.java | 1076 +++++
.../opennlp/tools/depparse/FeedforwardNetwork.java | 1110 +++++
.../java/opennlp/tools/depparse/Transition.java | 166 +
.../tools/dictionary/serializer/Attributes.java | 8 +-
.../tools/doccat/BagOfWordsFeatureGenerator.java | 9 +-
.../java/opennlp/tools/doccat/DoccatModel.java | 4 +-
.../tools/doccat/EmojiFeatureGenerator.java | 5 +-
.../tools/doccat/NGramFeatureGenerator.java | 9 +-
.../tools/entitylinker/EntityLinkerFactory.java | 4 +-
.../tools/langdetect/LanguageDetectorFactory.java | 4 +-
.../tools/lemmatizer/LemmatizerAnnotator.java | 18 +-
.../java/opennlp/tools/log/LogPrintStream.java | 8 +-
.../namefind/DefaultNameContextGenerator.java | 6 +-
.../tools/namefind/DictionaryNameFinder.java | 7 +-
.../tools/namefind/NameFinderAnnotator.java | 27 +-
.../opennlp/tools/namefind/RegexNameFinder.java | 24 +-
.../tools/namefind/RegexNameFinderFactory.java | 22 +-
.../tools/parser/AbstractBottomUpParser.java | 22 +-
.../java/opennlp/tools/parser/ParserAnnotator.java | 14 +-
.../java/opennlp/tools/parser/ParserFactory.java | 61 +-
.../java/opennlp/tools/parser/ParserModel.java | 10 +-
.../java/opennlp/tools/parser/chunking/Parser.java | 41 +-
.../parser/lang/es/AncoraSpanishHeadRules.java | 13 +-
.../opennlp/tools/parser/treeinsert/Parser.java | 44 +-
.../postag/ConfigurablePOSContextGenerator.java | 7 +-
.../java/opennlp/tools/postag/POSDictionary.java | 6 +-
.../main/java/opennlp/tools/postag/POSModel.java | 10 +-
.../opennlp/tools/postag/POSTagFormatMapper.java | 15 +-
.../opennlp/tools/postag/POSTaggerAnnotator.java | 8 +-
.../sentdetect/SentenceDetectorAnnotator.java | 12 +-
.../tools/sentdetect/SentenceDetectorME.java | 36 +-
.../opennlp/tools/sentdetect/SentenceModel.java | 4 +-
.../sentiment/EmojiSentimentContextGenerator.java | 5 +-
.../tools/sentiment/EmojiSentimentFactory.java | 2 +
.../java/opennlp/tools/stemmer/CachingStemmer.java | 18 +-
.../opennlp/tools/stemmer/DelegatingStemmer.java | 21 +-
.../java/opennlp/tools/stemmer/PorterStemmer.java | 14 +-
.../java/opennlp/tools/stemmer/SharingStemmer.java | 13 +-
.../opennlp/tools/stemmer/StemmerAnnotator.java | 8 +-
.../tools/stemmer/StemmerAnnotatorProvider.java | 9 +-
.../tools/stemmer/hunspell/AffixCondition.java | 20 +-
.../tools/stemmer/hunspell/HunspellDictionary.java | 126 +-
...nload.java => HunspellDictionaryInstaller.java} | 27 +-
.../tools/stemmer/hunspell/HunspellStemmer.java | 13 +-
.../stemmer/hunspell/HunspellStemmerFactory.java | 5 +-
.../tools/stemmer/snowball/SnowballStemmer.java | 3 +-
.../stemmer/snowball/SnowballStemmerFactory.java | 15 +-
.../tools/stopword/DictionaryStopwordFilter.java | 33 +-
.../tools/stopword/StopwordFilterStream.java | 12 +-
.../tools/stopword/StopwordFilteringTokenizer.java | 9 +-
.../java/opennlp/tools/stopword/StopwordLists.java | 73 +-
.../tools/termvector/TermVectorAnnotator.java | 59 +-
.../java/opennlp/tools/tokenize/BPETokenizer.java | 13 +-
.../tools/tokenize/BPETokenizerTrainer.java | 11 +-
.../opennlp/tools/tokenize/TokSpanEventStream.java | 4 +
.../opennlp/tools/tokenize/TokenSampleStream.java | 8 +-
.../opennlp/tools/tokenize/TokenizerAnnotator.java | 14 +-
.../tools/tokenize/TokenizerCharacterPolicy.java | 174 +
.../opennlp/tools/tokenize/TokenizerFactory.java | 7 +
.../java/opennlp/tools/tokenize/TokenizerME.java | 3 +
.../opennlp/tools/tokenize/TokenizerModel.java | 4 +-
.../java/opennlp/tools/tokenize/lang/Factory.java | 34 +-
.../tools/tokenize/lang/en/TokenSampleStream.java | 83 +-
.../tools/tokenize/lattice/CategoryTable.java | 15 +-
.../tools/tokenize/lattice/DoubleArrayLexicon.java | 10 +-
.../tools/tokenize/lattice/LatticeTokenizer.java | 26 +-
.../tools/tokenize/lattice/MecabDictionary.java | 223 +-
.../tokenize/lattice/MecabDictionaryInstaller.java | 180 +-
.../opennlp/tools/tokenize/lattice/Morpheme.java | 29 +-
.../tools/tokenize/lattice/UnigramSegmenter.java | 111 +-
.../tools/tokenize/lattice/WhitespaceRuns.java | 65 +
.../tools/tokenize/uax29/ExtendedPictographic.java | 99 +-
.../tools/tokenize/uax29/WordBreakProperty.java | 127 +-
.../tools/tokenize/uax29/WordSegmenter.java | 18 +-
.../tools/tokenize/uax29/WordTokenizer.java | 35 +-
.../opennlp/tools/tokenize/uax29/WordType.java | 14 +-
.../main/java/opennlp/tools/util/Checksums.java | 96 +
.../java/opennlp/tools/util/DictionaryCatalog.java | 91 +-
.../main/java/opennlp/tools/util/DownloadUtil.java | 285 +-
.../opennlp/tools/util/FilterObjectStream.java | 4 +-
.../java/opennlp/tools/util/ResourceInstaller.java | 338 +-
.../main/java/opennlp/tools/util/StringList.java | 3 +-
.../java/opennlp/tools/util/archive/TarStream.java | 74 +-
.../featuregen/AggregatedFeatureGenerator.java | 15 +-
.../EmojiAnnotationFeatureGenerator.java | 5 +-
.../EmojiAnnotationFeatureGeneratorFactory.java | 1 +
.../util/featuregen/FeatureGeneratorUtil.java | 63 +-
.../tools/util/featuregen/InSpanGenerator.java | 7 +-
.../java/opennlp/tools/util/model/BaseModel.java | 66 +-
.../tools/util/model/GenericModelSerializer.java | 11 +-
.../java/opennlp/tools/util/model/ModelUtil.java | 6 +-
.../AccentFoldCharSequenceNormalizer.java | 17 +-
.../AggregateCharSequenceNormalizer.java | 10 +-
.../AlignedAggregateCharSequenceNormalizer.java | 16 +-
.../opennlp/tools/util/normalizer/AsciiChars.java | 19 +
.../normalizer/BulletCharSequenceNormalizer.java | 14 +-
.../tools/util/normalizer/BundledUnicodeData.java | 204 +
.../normalizer/CaseFoldCharSequenceNormalizer.java | 18 +-
.../ConfusableSkeletonCharSequenceNormalizer.java | 11 +-
.../opennlp/tools/util/normalizer/Confusables.java | 121 +-
.../normalizer/DashCharSequenceNormalizer.java | 7 +
.../normalizer/DigitCharSequenceNormalizer.java | 8 +-
.../opennlp/tools/util/normalizer/Dimension.java | 10 +-
.../normalizer/EllipsisCharSequenceNormalizer.java | 5 +-
.../tools/util/normalizer/EmojiAnnotation.java | 19 +-
.../tools/util/normalizer/EmojiAnnotations.java | 127 +-
.../tools/util/normalizer/EmojiAnnotator.java | 19 +-
.../normalizer/EmojiCharSequenceNormalizer.java | 153 +-
.../tools/util/normalizer/EmojiEmoticons.java | 143 +-
.../tools/util/normalizer/EmojiEntityType.java | 11 +-
.../opennlp/tools/util/normalizer/EmojiFlags.java | 13 +-
.../EmojiToEmoticonCharSequenceNormalizer.java | 15 +-
.../EmoticonToEmojiCharSequenceNormalizer.java | 15 +-
.../FullCaseFoldCharSequenceNormalizer.java | 124 +-
.../GermanUmlautCharSequenceNormalizer.java | 10 +-
.../InvisibleCharSequenceNormalizer.java | 11 +-
...PreservingWhitespaceCharSequenceNormalizer.java | 8 +-
.../tools/util/normalizer/MailAddressScan.java | 28 +-
.../util/normalizer/NfcCharSequenceNormalizer.java | 13 +-
.../normalizer/NfkcCharSequenceNormalizer.java | 13 +-
.../util/normalizer/NormalizationProfile.java | 137 +-
.../util/normalizer/NormalizationProfiles.java | 54 +-
.../normalizer/NumberCharSequenceNormalizer.java | 6 +-
...PreservingWhitespaceCharSequenceNormalizer.java | 7 +-
.../normalizer/QuoteCharSequenceNormalizer.java | 14 +-
.../normalizer/ShrinkCharSequenceNormalizer.java | 13 +-
.../SocialMediaCharSequenceNormalizer.java | 11 +-
.../java/opennlp/tools/util/normalizer/Term.java | 13 +-
.../tools/util/normalizer/TermAnalyzer.java | 53 +-
.../tools/util/normalizer/TextNormalizer.java | 13 +-
.../util/normalizer/UnicodeEmojiSequences.java | 415 ++
.../util/normalizer/UrlCharSequenceNormalizer.java | 37 +-
.../WhitespaceCharSequenceNormalizer.java | 8 +-
.../tools/util/normalizer/EmojiSequences.txt | 5013 ++++++++++++++++++++
.../tools/util/normalizer/emoji-annotations.txt | 88 +-
.../opennlp/tools/AbstractModelLoaderTest.java | 4 +-
.../opennlp/tools/NullArgumentRejectionTest.java | 210 +
.../tools/chunker/ChunkerAnnotatorTest.java | 18 +-
.../tools/depparse/ArcStandardOracleTest.java | 195 +
.../tools/depparse/ArcStandardStateTest.java | 163 +
.../tools/depparse/AttachmentScoresTest.java | 77 +
.../depparse/DependencyContextGeneratorTest.java | 235 +
.../depparse/DependencyCrossValidatorTest.java | 268 ++
.../tools/depparse/DependencyEvaluatorTest.java | 250 +
.../depparse/DependencyParserEdgeCaseTest.java | 243 +
.../tools/depparse/DependencyParserMETest.java | 314 ++
.../tools/depparse/DependencyTestSamples.java | 340 ++
.../tools/depparse/FeedforwardContextTest.java | 116 +
.../depparse/FeedforwardDependencyParserTest.java | 1286 +++++
.../depparse/FeedforwardDependencyScoringTest.java | 417 ++
.../tools/depparse/FeedforwardRefinementTest.java | 217 +
.../opennlp/tools/depparse/RecordingMonitor.java | 48 +
.../opennlp/tools/depparse/TransitionTest.java | 70 +
.../doccat/BagOfWordsFeatureGeneratorTest.java | 4 +-
.../opennlp/tools/doccat/DocumentSampleTest.java | 9 +
.../tools/doccat/NGramFeatureGeneratorTest.java | 4 +-
.../tools/document/DocumentAnalyzerTest.java | 49 +-
.../document/DocumentPipelineExampleTest.java | 10 +-
...{TestComponents.java => DocumentTestStubs.java} | 25 +-
...efaultLanguageDetectorContextGeneratorTest.java | 60 +
.../tools/langdetect/LanguageSampleTest.java | 4 +-
.../opennlp/tools/langdetect/LanguageTest.java | 4 +-
.../tools/lemmatizer/DictionaryLemmatizerTest.java | 18 +-
.../tools/lemmatizer/LemmatizerMELocaleTest.java | 20 +-
.../tools/namefind/NameFinderMEWithDatesTest.java | 2 +-
.../opennlp/tools/namefind/NameSampleTest.java | 74 +
.../tools/parser/AbstractParserModelTest.java | 111 +
.../test/java/opennlp/tools/parser/ParseTest.java | 181 +
.../opennlp/tools/parser/ParserAnnotatorTest.java | 15 +-
.../opennlp/tools/parser/ParserFactoryTest.java | 207 +
.../java/opennlp/tools/parser/ParserTestUtil.java | 144 +
.../opennlp/tools/parser/chunking/ParserTest.java | 8 +
.../lang/es/AncoraSpanishHeadRulesBundledTest.java | 147 +
.../tools/parser/treeinsert/ParserTest.java | 8 +
.../opennlp/tools/postag/POSDictionaryTest.java | 10 +
.../SentenceDetectorMESpanMappingTest.java | 18 +-
.../opennlp/tools/stemmer/StemmerFactoryTest.java | 4 +
...t.java => HunspellDictionaryInstallerTest.java} | 12 +-
.../opennlp/tools/stopword/StopwordListsTest.java | 8 +
.../tools/termvector/SingleSpaceTokens.java | 46 +-
.../tools/termvector/TermVectorAnnotatorTest.java | 5 +
.../tools/termvector/TermVectorPipelineTest.java | 5 +-
.../tokenize/DefaultTokenContextGeneratorTest.java | 12 +-
.../tools/tokenize/SimpleTokenizerTest.java | 12 +-
.../opennlp/tools/tokenize/TokenSampleTest.java | 23 +
.../tokenize/TokenizerCharacterPolicyTest.java | 256 +
.../opennlp/tools/tokenize/TokenizerMETest.java | 6 -
.../tools/tokenize/WhitespaceTokenizerTest.java | 19 +-
.../tools/tokenize/WordpieceTokenizerTest.java | 29 +-
.../tokenize/lang/en/TokenSampleStreamTest.java | 133 +
.../tokenize/lattice/LatticeTokenizerTest.java | 184 +-
.../lattice/MecabDictionaryInstallerTest.java | 7 +-
.../tokenize/lattice/UnigramSegmenterTest.java | 50 +-
.../tokenize/uax29/ExtendedPictographicTest.java | 18 +
.../tokenize/uax29/WordBreakPropertyTest.java | 11 +-
.../tools/tokenize/uax29/WordSegmenterTest.java | 5 +-
.../tools/tokenize/uax29/WordTokenizerTest.java | 5 +-
.../opennlp/tools/tokenize/uax29/WordTypeTest.java | 5 +-
.../java/opennlp/tools/util/ChecksumsTest.java | 85 +
.../opennlp/tools/util/DictionaryCatalogTest.java | 87 +
.../tools/util/DownloadUtilCacheIntegrityTest.java | 22 +-
.../opennlp/tools/util/InstallerTestSupport.java | 42 -
.../tools/util/LanguageCodeValidatorTest.java | 64 +
.../tools/util/ResourceInstallerHttpTest.java | 16 +-
.../tools/util/ResourceInstallerSelectedTest.java | 198 +
.../opennlp/tools/util/ResourceInstallerTest.java | 106 +-
.../opennlp/tools/util/archive/TarStreamTest.java | 58 +
.../util/featuregen/FeatureGeneratorUtilTest.java | 74 +
.../opennlp/tools/util/model/BaseModelTest.java | 14 +
.../AccentFoldCharSequenceNormalizerTest.java | 5 +-
.../AggregateCharSequenceNormalizerTest.java | 4 +-
.../normalizer/AlignedNormalizerPipelineTest.java | 9 +-
.../util/normalizer/BundledUnicodeDataTest.java | 161 +
.../CaseFoldCharSequenceNormalizerTest.java | 9 +-
.../CharSequenceNormalizerContractTest.java | 33 +
.../util/normalizer/ConfusableSkeletonTest.java | 5 +-
.../util/normalizer/ConfusablesParseTest.java | 19 +
.../tools/util/normalizer/ConfusablesTest.java | 5 +-
.../util/normalizer/EmojiAnnotationsTest.java | 64 +-
...CharSequenceNormalizerCharacterizationTest.java | 203 +
.../EmojiCharSequenceNormalizerTest.java | 326 ++
.../EmojiToEmoticonCharSequenceNormalizerTest.java | 5 +-
.../EmoticonToEmojiCharSequenceNormalizerTest.java | 5 +-
.../FullCaseFoldCharSequenceNormalizerTest.java | 21 +-
.../GermanUmlautCharSequenceNormalizerTest.java | 5 +-
...ervingWhitespaceCharSequenceNormalizerTest.java | 5 +-
.../tools/util/normalizer/MailAddressScanTest.java | 52 +
.../util/normalizer/NormalizationProfilesTest.java | 75 +-
...ervingWhitespaceCharSequenceNormalizerTest.java | 5 +-
.../util/normalizer/SetBasedNormalizerTest.java | 5 +-
.../normalizer/TermAnalyzerMultilingualTest.java | 5 +-
.../tools/util/normalizer/TermAnalyzerTest.java | 9 +-
.../TermAnalyzerThreadLocalStateTest.java | 126 +
.../tools/util/normalizer/TextNormalizerTest.java | 5 +-
.../UnicodeCharSequenceNormalizerTest.java | 5 +-
.../normalizer/UnicodeEmojiSequencesParseTest.java | 100 +
.../resources/opennlp/tools/parser/es_head_rules | 22 +
opennlp-core/pom.xml | 20 +
opennlp-distr/README_FOOTER.html | 17 +
opennlp-distr/pom.xml | 26 +-
opennlp-distr/src/main/docker/Dockerfile | 17 +
opennlp-distr/src/main/readme/LICENSE | 3 +-
opennlp-distr/src/main/readme/NOTICE | 33 +-
opennlp-docs/src/docbkx/corpora.xml | 67 +-
opennlp-docs/src/docbkx/dependency.xml | 196 +
opennlp-docs/src/docbkx/doccat.xml | 12 +-
opennlp-docs/src/docbkx/document.xml | 45 +-
opennlp-docs/src/docbkx/introduction.xml | 57 +-
opennlp-docs/src/docbkx/langdetect.xml | 22 +-
opennlp-docs/src/docbkx/machine-learning.xml | 89 +-
opennlp-docs/src/docbkx/model-loading.xml | 15 +
opennlp-docs/src/docbkx/morfologik-addon.xml | 6 +
opennlp-docs/src/docbkx/namefinder.xml | 5 +-
opennlp-docs/src/docbkx/normalizer.xml | 85 +-
opennlp-docs/src/docbkx/opennlp.xml | 1 +
opennlp-docs/src/docbkx/parser.xml | 22 +
opennlp-docs/src/docbkx/spellcheck.xml | 14 +
opennlp-docs/src/docbkx/stemmer.xml | 32 +-
opennlp-docs/src/docbkx/tokenizer.xml | 88 +-
opennlp-eval-tests/lang/es/parser/es-head-rules | 24 +-
opennlp-eval-tests/pom.xml | 6 +
.../dl/doccat/DocumentCategorizerDLEval.java | 3 +-
.../opennlp/dl/namefinder/NameFinderDLEval.java | 98 +-
.../java/opennlp/tools/eval/AbstractEvalTest.java | 29 +
.../opennlp/tools/eval/ConllXPosTaggerEval.java | 2 +-
.../tools/eval/EnglishTokenSampleStreamEval.java | 226 +
.../tools/eval/HunspellClassificationEval.java | 80 +
.../tools/eval/HunspellCompatibilityEval.java | 43 +-
.../tools/eval/MultiThreadedStemmerEval.java | 94 +-
.../opennlp/tools/eval/SourceForgeModelEval.java | 23 +-
.../tools/eval/UniversalDependencyParserEval.java | 367 ++
opennlp-extensions/opennlp-morfologik/pom.xml | 7 +
.../builder/MorfologikDictionaryBuilder.java | 14 +-
.../lemmatizer/MorfologikLemmatizer.java | 2 +
.../tagdict/MorfologikTagDictionary.java | 37 +-
.../opennlp/morfologik/AbstractMorfologikTest.java | 6 +-
.../opennlp/morfologik/MorfologikLocaleTest.java | 19 +-
.../MorfologikTagDictionaryConcurrencyTest.java | 94 +
.../tagdict/MorfologikTagDictionaryTest.java | 39 +
...aryLocaleSafety.info => dictionaryTagless.info} | 0
.../src/test/resources/dictionaryTagless.txt | 2 +
.../main/java/opennlp/spellcheck/SpellChecker.java | 11 +-
.../main/java/opennlp/spellcheck/SuggestItem.java | 9 +-
.../dictionary/FrequencyDictionaryLoader.java | 86 +-
.../spellcheck/dictionary/SymSpellModel.java | 26 +-
.../dictionary/SymSpellModelResolver.java | 16 +-
.../dictionary/SymSpellModelSerializer.java | 12 +-
.../spellcheck/dictionary/SymSpellModels.java | 18 +-
.../SpellCheckingCharSequenceNormalizer.java | 215 +-
.../stream/SpellCorrectingObjectStream.java | 27 +-
.../stream/SpellCorrectingTokenStream.java | 39 +-
.../java/opennlp/spellcheck/symspell/SymSpell.java | 25 +-
.../spellcheck/symspell/SymSpellConfig.java | 6 +-
.../java/opennlp/spellcheck/SuggestItemTest.java | 4 +-
.../dictionary/FrequencyDictionaryLoaderTest.java | 18 +
.../dictionary/SymSpellModelResolverTest.java | 11 +
.../dictionary/SymSpellModelSerializationTest.java | 28 +
.../SymSpellModelSerializerLimitsTest.java | 8 +-
.../SpellCheckingCharSequenceNormalizerTest.java | 136 +-
.../normalizer/SpellCheckingUnicodeCaseTest.java | 116 +
.../stream/SpellCorrectingTokenStreamTest.java | 34 +
.../spellcheck/symspell/SymSpellCompoundTest.java | 2 +-
.../spellcheck/symspell/SymSpellConfigTest.java | 2 +-
.../spellcheck/symspell/SymSpellLookupTest.java | 14 +-
.../java/opennlp/uima/normalizer/NumberUtil.java | 7 +-
.../src/test/java/opennlp/uima/AbstractIT.java | 4 +-
.../opennlp/uima/normalizer/NumberUtilTest.java | 26 +-
opennlp-tools/lang/es/parser/es-head-rules | 24 +-
opennlp-tools/pom.xml | 7 +
.../tools/namefind/AbstractNameFinderTest.java | 4 +-
.../opennlp/tools/util/DownloadParserTest.java | 157 +-
.../java/opennlp/tools/util/DownloadUtilTest.java | 38 +
.../opennlp/tools/util/ObjectStreamUtilsTest.java | 21 +
.../tools/util/PlainTextByLineStreamTest.java | 9 +
.../opennlp/tools/util/TrainingParametersTest.java | 17 +-
.../tools/util/ext/ExtensionLoaderTest.java | 8 +-
.../EmojiCharSequenceNormalizerTest.java | 43 -
.../normalizer/UrlCharSequenceNormalizerTest.java | 2 +-
pom.xml | 66 +-
rat-excludes | 73 +-
src/license/NOTICE.template | 23 +-
569 files changed, 39508 insertions(+), 5482 deletions(-)
create mode 100644 dev/README-unicode-emoji-sequences.md
create mode 100644 dev/UnicodeEmojiSequenceGenerator.java
create mode 100644 dev/forbidden-regex.txt
create mode 100644
opennlp-api/src/main/java/opennlp/tools/commons/SuppressForbidden.java
create mode 100644
opennlp-api/src/main/java/opennlp/tools/depparse/DependencyArc.java
copy
opennlp-api/src/main/java/opennlp/tools/{lemmatizer/LemmatizerEvaluationMonitor.java
=> depparse/DependencyEvaluationMonitor.java} (79%)
create mode 100644
opennlp-api/src/main/java/opennlp/tools/depparse/DependencyParser.java
create mode 100644
opennlp-api/src/main/java/opennlp/tools/depparse/DependencySample.java
create mode 100644
opennlp-api/src/main/java/opennlp/tools/depparse/DependencyTree.java
create mode 100644
opennlp-api/src/main/java/opennlp/tools/document/DocumentMerges.java
create mode 100644
opennlp-api/src/main/java/opennlp/tools/tokenize/MappedText.java
create mode 100644
opennlp-api/src/main/java/opennlp/tools/util/ParamChecks.java
create mode 100644
opennlp-api/src/main/java/opennlp/tools/util/normalizer/HexCodePoints.java
copy opennlp-tools/src/test/java/opennlp/tools/HighMemoryUsage.java =>
opennlp-api/src/test/java/opennlp/tools/PinWhitespaceMode.java (56%)
create mode 100644
opennlp-api/src/test/java/opennlp/tools/PinWhitespaceModeTest.java
copy opennlp-tools/src/test/java/opennlp/tools/HighMemoryUsage.java =>
opennlp-api/src/test/java/opennlp/tools/ResetWhitespaceMode.java (58%)
create mode 100644
opennlp-api/src/test/java/opennlp/tools/ResetWhitespaceModeTest.java
create mode 100644
opennlp-api/src/test/java/opennlp/tools/RestoreDefaultLocale.java
create mode 100644
opennlp-api/src/test/java/opennlp/tools/RestoreDefaultLocaleTest.java
create mode 100644
opennlp-api/src/test/java/opennlp/tools/WhitespaceModeExtension.java
create mode 100644
opennlp-api/src/test/java/opennlp/tools/depparse/DependencySampleTest.java
create mode 100644
opennlp-api/src/test/java/opennlp/tools/depparse/DependencyTreeTest.java
create mode 100644
opennlp-api/src/test/java/opennlp/tools/tokenize/MappedTextTest.java
rename {opennlp-core/opennlp-runtime =>
opennlp-api}/src/test/java/opennlp/tools/tokenize/WordpieceEncoderReferenceSequencesTest.java
(100%)
rename {opennlp-core/opennlp-runtime =>
opennlp-api}/src/test/java/opennlp/tools/tokenize/WordpieceEncoderTest.java
(94%)
create mode 100644
opennlp-api/src/test/java/opennlp/tools/util/ParamChecksTest.java
create mode 100644
opennlp-api/src/test/java/opennlp/tools/util/StringUtilSplitNonEmptyTest.java
rename {opennlp-tools =>
opennlp-api}/src/test/java/opennlp/tools/util/StringUtilTest.java (77%)
copy
opennlp-tools/src/test/java/opennlp/tools/monitoring/IterDeltaAccuracyUnderToleranceTest.java
=> opennlp-api/src/test/java/opennlp/tools/util/StringUtilUpperCaseTest.java
(57%)
create mode 100644
opennlp-api/src/test/java/opennlp/tools/util/normalizer/HexCodePointsTest.java
copy
opennlp-core/opennlp-cli/src/main/java/opennlp/tools/cmdline/{tokenizer/DetokenEvaluationErrorListener.java
=> depparse/DependencyEvaluationErrorListener.java} (51%)
copy
opennlp-core/opennlp-cli/src/main/java/opennlp/tools/cmdline/{lemmatizer/LemmatizerModelLoader.java
=> depparse/DependencyModelLoader.java} (68%)
create mode 100644
opennlp-core/opennlp-cli/src/main/java/opennlp/tools/cmdline/depparse/DependencyParserCrossValidatorTool.java
create mode 100644
opennlp-core/opennlp-cli/src/main/java/opennlp/tools/cmdline/depparse/DependencyParserEvaluatorTool.java
copy
opennlp-core/opennlp-cli/src/main/java/opennlp/tools/cmdline/{tokenizer/TokenizerMETool.java
=> depparse/DependencyParserMETool.java} (61%)
create mode 100644
opennlp-core/opennlp-cli/src/main/java/opennlp/tools/cmdline/depparse/DependencyParserToolSupport.java
create mode 100644
opennlp-core/opennlp-cli/src/main/java/opennlp/tools/cmdline/depparse/DependencyParserTrainerTool.java
copy
opennlp-core/opennlp-cli/src/main/java/opennlp/tools/cmdline/{chunker/ChunkerModelLoader.java
=> depparse/FeedforwardDependencyModelLoader.java} (62%)
create mode 100644
opennlp-core/opennlp-cli/src/main/java/opennlp/tools/cmdline/depparse/FeedforwardDependencyParserEvaluatorTool.java
create mode 100644
opennlp-core/opennlp-cli/src/main/java/opennlp/tools/cmdline/depparse/FeedforwardDependencyParserTool.java
create mode 100644
opennlp-core/opennlp-cli/src/main/java/opennlp/tools/cmdline/depparse/FeedforwardDependencyParserTrainerTool.java
rename {opennlp-tools/src/test/java/opennlp/tools =>
opennlp-core/opennlp-cli/src/test/java/opennlp/tools/cmdline}/chunker/ChunkerDetailedFMeasureListenerTest.java
(73%)
copy opennlp-core/{opennlp-runtime/src/test/java/opennlp/tools =>
opennlp-cli/src/test/java/opennlp/tools/cmdline}/chunker/DummyChunkSampleStream.java
(96%)
copy {opennlp-tools/src/test/java/opennlp/tools =>
opennlp-core/opennlp-cli/src/test/java/opennlp/tools/cmdline}/chunker/DummyChunker.java
(94%)
create mode 100644
opennlp-core/opennlp-cli/src/test/java/opennlp/tools/cmdline/depparse/DependencyParserToolsTest.java
create mode 100644
opennlp-core/opennlp-cli/src/test/java/opennlp/tools/cmdline/depparse/DependencyToolTestSupport.java
create mode 100644
opennlp-core/opennlp-cli/src/test/java/opennlp/tools/cmdline/depparse/FeedforwardDependencyParserToolsTest.java
create mode 100644
opennlp-core/opennlp-cli/src/test/java/opennlp/tools/cmdline/parser/ParserToolTest.java
rename {opennlp-tools/src/test/resources/opennlp/tools =>
opennlp-core/opennlp-cli/src/test/resources/opennlp/tools/cmdline}/chunker/detailedOutput.txt
(100%)
copy opennlp-core/{opennlp-runtime/src/test/resources/opennlp/tools =>
opennlp-cli/src/test/resources/opennlp/tools/cmdline}/chunker/output.txt (100%)
create mode 100644
opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/conllu/ConlluDependencySampleStream.java
create mode 100644
opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/conllu/ConlluDependencySampleStreamFactory.java
create mode 100644
opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/masc/MascIdentifiers.java
create mode 100644
opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/conllu/ConlluDependencyParserUsageTest.java
create mode 100644
opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/conllu/ConlluDependencySampleStreamFactoryTest.java
create mode 100644
opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/conllu/ConlluDependencySampleStreamTest.java
create mode 100644
opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/conllu/ConlluTagsetTest.java
copy
opennlp-core/opennlp-formats/src/{main/java/opennlp/tools/formats/conllu/ConlluTagset.java
=> test/java/opennlp/tools/formats/conllu/ConlluTestLines.java} (64%)
create mode 100644
opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/masc/MascDocumentTest.java
create mode 100644
opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/masc/MascIdentifiersTest.java
create mode 100644
opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/masc/MascNamedEntityParserTest.java
create mode 100644
opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/masc/MascParserTestUtil.java
create mode 100644
opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/masc/MascPennTagParserTest.java
create mode 100644
opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/masc/MascSentenceParserTest.java
create mode 100644
opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/masc/MascWordParserTest.java
create mode 100644
opennlp-core/opennlp-ml/opennlp-dl/src/test/java/opennlp/dl/AbstractDLColdStartCudaTest.java
create mode 100644
opennlp-core/opennlp-ml/opennlp-dl/src/test/java/opennlp/dl/AbstractDLSoftmaxTest.java
create mode 100644
opennlp-core/opennlp-ml/opennlp-dl/src/test/java/opennlp/dl/GpuRequestTest.java
create mode 100644
opennlp-core/opennlp-ml/opennlp-dl/src/test/java/opennlp/dl/OnnxTestSupport.java
create mode 100644
opennlp-core/opennlp-ml/opennlp-dl/src/test/java/opennlp/dl/vectors/SentenceVectorsDLBatchLimitTest.java
create mode 100644
opennlp-core/opennlp-ml/opennlp-dl/src/test/java/opennlp/dl/vectors/SentenceVectorsDLInferenceOptionsTest.java
create mode 100644
opennlp-core/opennlp-ml/opennlp-dl/src/test/java/opennlp/dl/vectors/TinyVectorModels.java
create mode 100644
opennlp-core/opennlp-ml/opennlp-dl/src/test/resources/opennlp/dl/tokenizer.json
create mode 100644
opennlp-core/opennlp-ml/opennlp-ml-commons/src/main/java/opennlp/tools/ml/model/EventFields.java
create mode 100644
opennlp-core/opennlp-ml/opennlp-ml-commons/src/test/java/opennlp/tools/ml/model/AbstractModelTest.java
create mode 100644
opennlp-core/opennlp-ml/opennlp-ml-commons/src/test/java/opennlp/tools/ml/model/SimpleEventStreamBuilderTest.java
create mode 100644
opennlp-core/opennlp-ml/opennlp-ml-maxent/src/test/java/opennlp/tools/ml/maxent/BasicContextGeneratorTest.java
create mode 100644
opennlp-core/opennlp-model-resolver/src/main/java/opennlp/tools/models/WildcardMatcher.java
create mode 100644
opennlp-core/opennlp-model-resolver/src/test/java/opennlp/tools/models/AbstractClassPathModelFinderConcurrencyTest.java
create mode 100644
opennlp-core/opennlp-model-resolver/src/test/java/opennlp/tools/models/AbstractClassPathModelFinderTest.java
create mode 100644
opennlp-core/opennlp-model-resolver/src/test/java/opennlp/tools/models/ModelFinderPathTest.java
create mode 100644
opennlp-core/opennlp-model-resolver/src/test/java/opennlp/tools/models/UnescapedUrls.java
create mode 100644
opennlp-core/opennlp-model-resolver/src/test/java/opennlp/tools/models/WildcardMatcherTest.java
create mode 100644
opennlp-core/opennlp-model-resolver/src/test/java/opennlp/tools/models/dir/DirectoryModelFinderTest.java
create mode 100644
opennlp-core/opennlp-runtime/src/main/appended-resources/META-INF/LICENSE
create mode 100644
opennlp-core/opennlp-runtime/src/main/appended-resources/META-INF/NOTICE
create mode 100644
opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/ArcStandardBeam.java
create mode 100644
opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/ArcStandardOracle.java
create mode 100644
opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/ArcStandardState.java
create mode 100644
opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/AttachmentScores.java
create mode 100644
opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/ContributionCache.java
create mode 100644
opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyContextGenerator.java
create mode 100644
opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyCrossValidator.java
create mode 100644
opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyEvaluator.java
create mode 100644
opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyEventStream.java
create mode 100644
opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyModel.java
create mode 100644
opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyParserME.java
create mode 100644
opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardContext.java
create mode 100644
opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyModel.java
create mode 100644
opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyParser.java
create mode 100644
opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyTrainer.java
create mode 100644
opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardNetwork.java
create mode 100644
opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/Transition.java
rename
opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/{HunspellDictionaryDownload.java
=> HunspellDictionaryInstaller.java} (75%)
create mode 100644
opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/TokenizerCharacterPolicy.java
rename {opennlp-api =>
opennlp-core/opennlp-runtime}/src/main/java/opennlp/tools/tokenize/lattice/CategoryTable.java
(95%)
rename {opennlp-api =>
opennlp-core/opennlp-runtime}/src/main/java/opennlp/tools/tokenize/lattice/DoubleArrayLexicon.java
(96%)
rename {opennlp-api =>
opennlp-core/opennlp-runtime}/src/main/java/opennlp/tools/tokenize/lattice/LatticeTokenizer.java
(95%)
rename {opennlp-api =>
opennlp-core/opennlp-runtime}/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionary.java
(78%)
rename {opennlp-api =>
opennlp-core/opennlp-runtime}/src/main/java/opennlp/tools/tokenize/lattice/Morpheme.java
(70%)
rename {opennlp-api =>
opennlp-core/opennlp-runtime}/src/main/java/opennlp/tools/tokenize/lattice/UnigramSegmenter.java
(81%)
create mode 100644
opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/WhitespaceRuns.java
create mode 100644
opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/Checksums.java
create mode 100644
opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/BundledUnicodeData.java
create mode 100644
opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/UnicodeEmojiSequences.java
create mode 100644
opennlp-core/opennlp-runtime/src/main/resources/opennlp/tools/util/normalizer/EmojiSequences.txt
create mode 100644
opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/NullArgumentRejectionTest.java
create mode 100644
opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/ArcStandardOracleTest.java
create mode 100644
opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/ArcStandardStateTest.java
create mode 100644
opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/AttachmentScoresTest.java
create mode 100644
opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyContextGeneratorTest.java
create mode 100644
opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyCrossValidatorTest.java
create mode 100644
opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyEvaluatorTest.java
create mode 100644
opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyParserEdgeCaseTest.java
create mode 100644
opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyParserMETest.java
create mode 100644
opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyTestSamples.java
create mode 100644
opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/FeedforwardContextTest.java
create mode 100644
opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/FeedforwardDependencyParserTest.java
create mode 100644
opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/FeedforwardDependencyScoringTest.java
create mode 100644
opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/FeedforwardRefinementTest.java
create mode 100644
opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/RecordingMonitor.java
create mode 100644
opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/TransitionTest.java
rename
opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/document/{TestComponents.java
=> DocumentTestStubs.java} (76%)
create mode 100644
opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/parser/ParserFactoryTest.java
create mode 100644
opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/parser/lang/es/AncoraSpanishHeadRulesBundledTest.java
rename
opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/hunspell/{HunspellDictionaryDownloadTest.java
=> HunspellDictionaryInstallerTest.java} (94%)
create mode 100644
opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/TokenizerCharacterPolicyTest.java
create mode 100644
opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lang/en/TokenSampleStreamTest.java
create mode 100644
opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/ChecksumsTest.java
create mode 100644
opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/ResourceInstallerSelectedTest.java
create mode 100644
opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/BundledUnicodeDataTest.java
create mode 100644
opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/EmojiCharSequenceNormalizerCharacterizationTest.java
create mode 100644
opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/EmojiCharSequenceNormalizerTest.java
create mode 100644
opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/MailAddressScanTest.java
create mode 100644
opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/TermAnalyzerThreadLocalStateTest.java
create mode 100644
opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/UnicodeEmojiSequencesParseTest.java
create mode 100644
opennlp-core/opennlp-runtime/src/test/resources/opennlp/tools/parser/es_head_rules
create mode 100644 opennlp-docs/src/docbkx/dependency.xml
create mode 100644
opennlp-eval-tests/src/test/java/opennlp/tools/eval/EnglishTokenSampleStreamEval.java
create mode 100644
opennlp-eval-tests/src/test/java/opennlp/tools/eval/HunspellClassificationEval.java
create mode 100644
opennlp-eval-tests/src/test/java/opennlp/tools/eval/UniversalDependencyParserEval.java
create mode 100644
opennlp-extensions/opennlp-morfologik/src/test/java/opennlp/morfologik/tagdict/MorfologikTagDictionaryConcurrencyTest.java
copy
opennlp-extensions/opennlp-morfologik/src/test/resources/{dictionaryLocaleSafety.info
=> dictionaryTagless.info} (100%)
create mode 100644
opennlp-extensions/opennlp-morfologik/src/test/resources/dictionaryTagless.txt
create mode 100644
opennlp-extensions/opennlp-spellcheck/src/test/java/opennlp/spellcheck/normalizer/SpellCheckingUnicodeCaseTest.java
delete mode 100644
opennlp-tools/src/test/java/opennlp/tools/util/normalizer/EmojiCharSequenceNormalizerTest.java
