This is an automated email from the ASF dual-hosted git repository.
krickert pushed a change to branch OPENNLP-1888-DocumentShape
in repository https://gitbox.apache.org/repos/asf/opennlp.git
from 47367ddf8 OPENNLP-1233: Link the POS and chunk tag references (#1170)
add 64a8439a5 OPENNLP-587: Document the Entity Linker framework (#1169)
add 7cfd7f2dc Change the notification pattern so dev list is not flooded
with emails. (#1176)
add b4f52fa02 OPENNLP-244: Document the BioNLP converter (#1175)
add cf83a1375 OPENNLP-1742: Use the MaxEnt training default for
NameFinderME (#1171)
add 725b1f307 OPENNLP-1868: Add Unicode full case folding to text
normalization (#1138)
add dfd0c94db OPENNLP-1869: Support offset-aware bidirectional emoji and
emoticon expansion and contraction (#1164)
add 9796b1a09 Allowlist based class Instantiation (#1178)
add 4a435d3c3 OPENNLP-1890: Using allowlist based class loading in
StreamFactoryRegistry
add fa79a5243 OPENNLP-1890: Using allowlist based class loading in
StringInterners
add db7adf0a6 OPENNLP-1876: Replace regex with cursor scans in the legacy
CharSequenceNormalizers (#1151)
new 29149f4d4 OPENNLP-1888: Document annotation container: typed
offset-anchored layers over the original text
new eec278db5 OPENNLP-1888: Lemma and stem layer adapters
new 53a15bcb0 OPENNLP-1888: Pipeline example and contract tests for the
document container, javadoc precision pass
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.asf.yaml | 9 +-
LICENSE | 2 +-
NOTICE | 3 +
.../java/opennlp/tools/document/Annotation.java | 55 +
.../main/java/opennlp/tools/document/Document.java | 87 +
.../opennlp/tools/document/DocumentAnalyzer.java | 116 ++
.../opennlp/tools/document/DocumentAnnotator.java | 59 +
.../opennlp/tools/document/ImmutableDocument.java | 108 ++
.../main/java/opennlp/tools/document/LayerKey.java | 102 ++
.../main/java/opennlp/tools/document/Layers.java | 55 +
.../tools/document/NameFinderAnnotator.java | 88 +
.../opennlp/tools/document/POSTaggerAnnotator.java | 80 +
.../tools/document/SentenceDetectorAnnotator.java | 70 +
.../opennlp/tools/document/TokenizerAnnotator.java | 84 +
.../util/normalizer/CharSequenceNormalizer.java | 3 +-
.../util/normalizer/OffsetAwareNormalizer.java | 1 +
.../tools/document/DocumentAnalyzerTest.java | 194 +++
.../tools/document/DocumentContractTest.java | 281 ++++
.../document/DocumentPipelineExampleTest.java | 290 ++++
.../java/opennlp/tools/document/DocumentTest.java | 125 ++
.../tools/document/NameFinderAnnotatorTest.java | 87 +
.../EntityLinkerManualExamplesTest.java | 177 ++
.../tools/cmdline/TokenNameFinderToolTest.java | 3 +
.../tools/cmdline/StreamFactoryRegistry.java | 27 +-
.../tools/cmdline/StreamFactoryRegistryTest.java | 82 +
.../BioNLP2004NameSampleStreamFactoryTest.java | 24 +
.../tools/langdetect/LanguageDetectorFactory.java | 19 +-
.../tools/lemmatizer/LemmatizerAnnotator.java | 97 ++
.../java/opennlp/tools/namefind/NameFinderME.java | 4 +-
.../opennlp/tools/stemmer/StemmerAnnotator.java | 84 +
.../tools/util/featuregen/GeneratorFactory.java | 45 +-
.../tools/util/jvm/CHMStringDeduplicator.java | 2 +-
.../opennlp/tools/util/jvm/CHMStringInterner.java | 2 +-
.../opennlp/tools/util/jvm/HMStringInterner.java | 2 +-
.../opennlp/tools/util/jvm/JvmStringInterner.java | 2 +-
.../opennlp/tools/util/jvm/NoOpStringInterner.java | 2 +-
.../opennlp/tools/util/jvm/StringInterners.java | 12 +-
.../AccentFoldCharSequenceNormalizer.java | 6 +-
.../AggregateCharSequenceNormalizer.java | 4 +
.../opennlp/tools/util/normalizer/AsciiChars.java | 77 +
.../normalizer/CaseFoldCharSequenceNormalizer.java | 6 +-
.../ConfusableSkeletonCharSequenceNormalizer.java | 6 +-
.../opennlp/tools/util/normalizer/Dimension.java | 20 +-
.../normalizer/EmojiCharSequenceNormalizer.java | 8 +
.../tools/util/normalizer/EmojiEmoticons.java | 442 +++++
.../EmojiToEmoticonCharSequenceNormalizer.java | 63 +
.../EmoticonToEmojiCharSequenceNormalizer.java | 64 +
.../FullCaseFoldCharSequenceNormalizer.java | 193 +++
.../util/normalizer/NfcCharSequenceNormalizer.java | 6 +-
.../normalizer/NfkcCharSequenceNormalizer.java | 6 +-
.../normalizer/NumberCharSequenceNormalizer.java | 33 +-
.../normalizer/ShrinkCharSequenceNormalizer.java | 128 +-
.../SocialMediaCharSequenceNormalizer.java | 306 ++++
.../tools/util/normalizer/TermAnalyzer.java | 56 +
.../tools/util/normalizer/TextNormalizer.java | 80 +-
.../normalizer/TwitterCharSequenceNormalizer.java | 55 -
.../util/normalizer/UrlCharSequenceNormalizer.java | 192 ++-
.../opennlp/tools/util/normalizer/CaseFolding.txt | 1682 ++++++++++++++++++++
.../tools/util/normalizer/emoji-emoticons.txt | 90 ++
.../tools/lemmatizer/LemmatizerAnnotatorTest.java | 82 +
.../opennlp/tools/namefind/NameFinderMETest.java | 3 +
.../tools/namefind/NameFinderMEWithDatesTest.java | 18 +-
.../tools/stemmer/StemmerAnnotatorTest.java | 56 +
.../util/featuregen/GeneratorFactoryTest.java | 46 +-
.../tools/util/jvm/StringInternersTest.java | 74 +
.../normalizer/AlignedNormalizerPipelineTest.java | 8 +-
.../CharSequenceNormalizerContractTest.java | 88 +
.../util/normalizer/CharacterizationInputs.java | 51 +
.../tools/util/normalizer/EmojiEmoticonsTest.java | 181 +++
.../EmojiToEmoticonCharSequenceNormalizerTest.java | 164 ++
.../EmoticonToEmojiCharSequenceNormalizerTest.java | 135 ++
.../FullCaseFoldCharSequenceNormalizerTest.java | 275 ++++
...CharSequenceNormalizerCharacterizationTest.java | 102 ++
...CharSequenceNormalizerCharacterizationTest.java | 166 ++
...CharSequenceNormalizerCharacterizationTest.java | 198 +++
.../tools/util/normalizer/TermAnalyzerTest.java | 76 +
.../tools/util/normalizer/TextNormalizerTest.java | 29 +-
...CharSequenceNormalizerCharacterizationTest.java | 204 +++
opennlp-distr/src/main/readme/LICENSE | 1 +
opennlp-distr/src/main/readme/NOTICE | 3 +
opennlp-docs/src/docbkx/cli.xml | 8 +
opennlp-docs/src/docbkx/corpora.xml | 74 +
opennlp-docs/src/docbkx/entitylinker.xml | 213 +++
opennlp-docs/src/docbkx/normalizer.xml | 35 +-
opennlp-docs/src/docbkx/opennlp.xml | 1 +
.../SpellCheckingCharSequenceNormalizer.java | 192 ++-
.../stream/SpellCorrectingObjectStream.java | 31 +-
.../stream/SpellCorrectingTokenStream.java | 67 +-
.../SpellCheckingCharSequenceNormalizerTest.java | 74 +-
.../stream/SpellCorrectingObjectStreamTest.java | 16 +-
.../stream/SpellCorrectingTokenStreamTest.java | 47 +
.../TwitterCharSequenceNormalizerTest.java | 61 -
src/license/NOTICE.template | 3 +
93 files changed, 8433 insertions(+), 355 deletions(-)
create mode 100644
opennlp-api/src/main/java/opennlp/tools/document/Annotation.java
create mode 100644
opennlp-api/src/main/java/opennlp/tools/document/Document.java
create mode 100644
opennlp-api/src/main/java/opennlp/tools/document/DocumentAnalyzer.java
create mode 100644
opennlp-api/src/main/java/opennlp/tools/document/DocumentAnnotator.java
create mode 100644
opennlp-api/src/main/java/opennlp/tools/document/ImmutableDocument.java
create mode 100644
opennlp-api/src/main/java/opennlp/tools/document/LayerKey.java
create mode 100644 opennlp-api/src/main/java/opennlp/tools/document/Layers.java
create mode 100644
opennlp-api/src/main/java/opennlp/tools/document/NameFinderAnnotator.java
create mode 100644
opennlp-api/src/main/java/opennlp/tools/document/POSTaggerAnnotator.java
create mode 100644
opennlp-api/src/main/java/opennlp/tools/document/SentenceDetectorAnnotator.java
create mode 100644
opennlp-api/src/main/java/opennlp/tools/document/TokenizerAnnotator.java
create mode 100644
opennlp-api/src/test/java/opennlp/tools/document/DocumentAnalyzerTest.java
create mode 100644
opennlp-api/src/test/java/opennlp/tools/document/DocumentContractTest.java
create mode 100644
opennlp-api/src/test/java/opennlp/tools/document/DocumentPipelineExampleTest.java
create mode 100644
opennlp-api/src/test/java/opennlp/tools/document/DocumentTest.java
create mode 100644
opennlp-api/src/test/java/opennlp/tools/document/NameFinderAnnotatorTest.java
create mode 100644
opennlp-api/src/test/java/opennlp/tools/entitylinker/EntityLinkerManualExamplesTest.java
create mode 100644
opennlp-core/opennlp-formats/src/test/java/opennlp/tools/cmdline/StreamFactoryRegistryTest.java
create mode 100644
opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/lemmatizer/LemmatizerAnnotator.java
create mode 100644
opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/StemmerAnnotator.java
create mode 100644
opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/AsciiChars.java
create mode 100644
opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/EmojiEmoticons.java
create mode 100644
opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/EmojiToEmoticonCharSequenceNormalizer.java
create mode 100644
opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/EmoticonToEmojiCharSequenceNormalizer.java
create mode 100644
opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/FullCaseFoldCharSequenceNormalizer.java
create mode 100644
opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/SocialMediaCharSequenceNormalizer.java
delete mode 100644
opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/TwitterCharSequenceNormalizer.java
create mode 100644
opennlp-core/opennlp-runtime/src/main/resources/opennlp/tools/util/normalizer/CaseFolding.txt
create mode 100644
opennlp-core/opennlp-runtime/src/main/resources/opennlp/tools/util/normalizer/emoji-emoticons.txt
create mode 100644
opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/lemmatizer/LemmatizerAnnotatorTest.java
create mode 100644
opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/StemmerAnnotatorTest.java
create mode 100644
opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/jvm/StringInternersTest.java
create mode 100644
opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/CharSequenceNormalizerContractTest.java
create mode 100644
opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/CharacterizationInputs.java
create mode 100644
opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/EmojiEmoticonsTest.java
create mode 100644
opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/EmojiToEmoticonCharSequenceNormalizerTest.java
create mode 100644
opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/EmoticonToEmojiCharSequenceNormalizerTest.java
create mode 100644
opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/FullCaseFoldCharSequenceNormalizerTest.java
create mode 100644
opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/NumberCharSequenceNormalizerCharacterizationTest.java
create mode 100644
opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/ShrinkCharSequenceNormalizerCharacterizationTest.java
create mode 100644
opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/SocialMediaCharSequenceNormalizerCharacterizationTest.java
create mode 100644
opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/UrlCharSequenceNormalizerCharacterizationTest.java
create mode 100644 opennlp-docs/src/docbkx/entitylinker.xml
delete mode 100644
opennlp-tools/src/test/java/opennlp/tools/util/normalizer/TwitterCharSequenceNormalizerTest.java
