This is an automated email from the ASF dual-hosted git repository.
krickert pushed a change to branch OPENNLP-1875-whitespace-ucd
in repository https://gitbox.apache.org/repos/asf/opennlp.git
from d22e390da OPENNLP-1875: Share the line separator check via StringUtil
and clarify the legacy predicate notes
add 2fdad676b OPENNLP-1875: Refresh the Leipzig training-data fingerprint
for the White_Space tokenizer
add a58effa4a OPENNLP-1875: Narrow to user-text paths; keep the
trained-model tokenizers on legacy whitespace
add 61498f6fa OPENNLP-1875: Switch whitespace classification to Unicode
White_Space with a legacy compatibility mode
No new revisions were added by this update.
Summary of changes:
.../tools/tokenize/WhitespaceTokenizer.java | 12 +-
.../src/main/java/opennlp/tools/util/Span.java | 7 +-
.../main/java/opennlp/tools/util/StringUtil.java | 69 +++++------
.../java/opennlp/tools/util/WhitespaceMode.java | 134 +++++++++++++++++++++
.../tokenize/BertNormalizationWhitespaceTest.java | 58 ---------
.../opennlp/tools/util/WhitespaceModeTest.java | 130 ++++++++++++++++++++
.../tools/cmdline/stopword/StopwordFilterTool.java | 3 +-
.../cmdline/stopword/StopwordFilterToolTest.java | 35 ------
.../sentdetect/DefaultSDContextGenerator.java | 5 +
.../tools/sentdetect/SentenceDetectorME.java | 12 +-
.../tokenize/DefaultTokenContextGenerator.java | 7 +-
.../opennlp/tools/tokenize/SimpleTokenizer.java | 17 ++-
.../opennlp/tools/namefind/NameSampleTest.java | 30 -----
.../java/opennlp/tools/postag/POSSampleTest.java | 14 ---
.../sentdetect/DefaultSDContextGeneratorTest.java | 29 -----
.../SentenceDetectorMEAbbreviationBreakTest.java | 101 ----------------
.../SentenceDetectorMESpanMappingTest.java | 14 +++
.../tokenize/DefaultTokenContextGeneratorTest.java | 62 +++++++---
.../tools/tokenize/SimpleTokenizerTest.java | 116 ++++++------------
.../opennlp/tools/tokenize/TokenSampleTest.java | 18 ---
.../tools/tokenize/WhitespaceTokenizerTest.java | 116 ++++++------------
opennlp-docs/src/docbkx/introduction.xml | 23 ++++
opennlp-docs/src/docbkx/tokenizer.xml | 5 +
.../opennlp/tools/eval/SourceForgeModelEval.java | 19 +++
.../spellcheck/cmdline/CorrectTextTool.java | 7 +-
.../cmdline/SpellCheckCmdLineToolsTest.java | 72 -----------
.../src/test/java/opennlp/tools/util/SpanTest.java | 38 ------
.../java/opennlp/tools/util/StringUtilTest.java | 67 +++++++++--
28 files changed, 571 insertions(+), 649 deletions(-)
create mode 100644
opennlp-api/src/main/java/opennlp/tools/util/WhitespaceMode.java
delete mode 100644
opennlp-api/src/test/java/opennlp/tools/tokenize/BertNormalizationWhitespaceTest.java
create mode 100644
opennlp-api/src/test/java/opennlp/tools/util/WhitespaceModeTest.java
delete mode 100644
opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/sentdetect/SentenceDetectorMEAbbreviationBreakTest.java
