This is an automated email from the ASF dual-hosted git repository.
krickert pushed a change to branch OPENNLP-205
in repository https://gitbox.apache.org/repos/asf/opennlp.git
discard 559ed6572 OPENNLP-205: Review follow-up: probs contract, code-point
cursors, detection-loop scope
discard f5b2189be OPENNLP-205: Map end-of-sentence positions to spans through
the CharClass whitespace engine
discard 3a8b23ce1 OPENNLP-205: Characterize the sentence position to span
mapping
add 7407f2419 Bump actions/download-artifact from 4.3.0 to 8.0.1
add d76737daa Bump actions/upload-artifact from 4.6.2 to 7.0.1
add a485a8dc8 OPENNLP-1861: Offset/alignment layer — Alignment,
AlignedText, buildAligned (1b/7) (#1109)
add c3d3ffd57 OPENNLP-1871: Add the stopword-list BSD section to the
binary distribution LICENSE (#1140)
add 47fd462a0 OPENNLP-1859: Add tests for BilouCodec encode/decode and
outcome compatibility (#1135)
add e2ffecd8a OPENNLP-1862: UAX #29 word tokenizer — WordSegmenter,
WordTokenizer, WordType (#1110)
add 2260b55ec Minor: Regenerated NOTICE File for
e2ffecd8a278653969d39045dff4dccfbdc9569c (#1142)
add 74092e1c9 OPENNLP-1873: Update ONNX runtime to 1.27.0 (#1143)
add 1712f32db OPENNLP-1874: Update Log4J to 2.26.1 (#1144)
add 945a3079f Minor: Regenerated NOTICE File for
1712f32dbe657792f9a80fa960018ab2a9eb2206 (#1147)
add a72109766 OPENNLP-205: Characterize the sentence position to span
mapping
add 696130547 OPENNLP-205: Map end-of-sentence positions to spans through
the CharClass whitespace engine
add 42012548b OPENNLP-205: Review follow-up: probs contract, code-point
cursors, detection-loop scope
add f59954a6b OPENNLP-205 Review: make the span-mapping helpers instance
methods
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (559ed6572)
\
N -- N -- N refs/heads/OPENNLP-205 (f59954a6b)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
No new revisions were added by this update.
Summary of changes:
.github/workflows/maven.yml | 8 +-
LICENSE | 8 +-
NOTICE | 38 +-
.../opennlp/tools/util/normalizer/AlignedText.java | 71 +
.../opennlp/tools/util/normalizer/Alignment.java | 298 +++
.../opennlp/tools/util/normalizer/CharClass.java | 253 ++-
.../util/normalizer/OffsetAwareNormalizer.java | 49 +
.../tools/util/normalizer/AlignedTextTest.java | 93 +
.../tools/util/normalizer/AlignmentTest.java | 258 +++
.../tools/util/normalizer/CharClassTest.java | 214 +++
.../tools/sentdetect/SentenceDetectorME.java | 13 +-
.../tools/tokenize/uax29/ExtendedPictographic.java | 146 ++
.../opennlp/tools/tokenize/uax29/WordBreak.java | 99 +
.../tools/tokenize/uax29/WordBreakProperty.java | 261 +++
.../tools/tokenize/uax29/WordSegmenter.java | 416 +++++
.../opennlp/tools/tokenize/uax29/WordToken.java | 29 +-
.../tools/tokenize/uax29/WordTokenizer.java | 203 ++
.../opennlp/tools/tokenize/uax29/WordType.java | 166 ++
.../AlignedAggregateCharSequenceNormalizer.java | 67 +
.../normalizer/BulletCharSequenceNormalizer.java | 8 +-
.../normalizer/DashCharSequenceNormalizer.java | 8 +-
.../normalizer/DigitCharSequenceNormalizer.java | 8 +-
.../normalizer/EllipsisCharSequenceNormalizer.java | 8 +-
.../GermanUmlautCharSequenceNormalizer.java | 9 +-
.../InvisibleCharSequenceNormalizer.java | 8 +-
...PreservingWhitespaceCharSequenceNormalizer.java | 72 +
.../normalizer/QuoteCharSequenceNormalizer.java | 11 +-
.../tools/util/normalizer/TextNormalizer.java | 41 +
.../WhitespaceCharSequenceNormalizer.java | 11 +-
.../tools/tokenize/uax29/ExtendedPictographic.txt | 461 +++++
.../tools/tokenize/uax29/WordBreakProperty.txt | 1541 +++++++++++++++
.../opennlp/tools/namefind/BilouCodecTest.java | 381 ++++
.../SentenceDetectorMESpanMappingTest.java | 22 +-
.../tokenize/uax29/ExtendedPictographicTest.java | 74 +
.../uax29/WordBoundaryConformanceTest.java | 112 ++
.../tokenize/uax29/WordBreakPropertyTest.java | 131 ++
.../tools/tokenize/uax29/WordBreakTest.java | 75 +
.../tools/tokenize/uax29/WordSegmenterTest.java | 110 ++
.../tools/tokenize/uax29/WordTokenizerTest.java | 187 ++
.../opennlp/tools/tokenize/uax29/WordTypeTest.java | 93 +
.../normalizer/AlignedNormalizerPipelineTest.java | 342 ++++
.../GermanUmlautCharSequenceNormalizerTest.java | 11 +
...ervingWhitespaceCharSequenceNormalizerTest.java | 97 +
.../opennlp/tools/tokenize/uax29/WordBreakTest.txt | 1974 ++++++++++++++++++++
opennlp-distr/src/main/readme/LICENSE | 35 +
opennlp-distr/src/main/readme/NOTICE | 38 +-
pom.xml | 4 +-
rat-excludes | 5 +-
src/license/NOTICE.template | 28 +-
49 files changed, 8494 insertions(+), 101 deletions(-)
create mode 100644
opennlp-api/src/main/java/opennlp/tools/util/normalizer/AlignedText.java
create mode 100644
opennlp-api/src/main/java/opennlp/tools/util/normalizer/Alignment.java
create mode 100644
opennlp-api/src/main/java/opennlp/tools/util/normalizer/OffsetAwareNormalizer.java
create mode 100644
opennlp-api/src/test/java/opennlp/tools/util/normalizer/AlignedTextTest.java
create mode 100644
opennlp-api/src/test/java/opennlp/tools/util/normalizer/AlignmentTest.java
create mode 100644
opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/uax29/ExtendedPictographic.java
create mode 100644
opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/uax29/WordBreak.java
create mode 100644
opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/uax29/WordBreakProperty.java
create mode 100644
opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/uax29/WordSegmenter.java
copy opennlp-api/src/main/java/opennlp/tools/tokenize/AbstractTokenizer.java
=>
opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/uax29/WordToken.java
(60%)
create mode 100644
opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/uax29/WordTokenizer.java
create mode 100644
opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/uax29/WordType.java
create mode 100644
opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/AlignedAggregateCharSequenceNormalizer.java
create mode 100644
opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/LineBreakPreservingWhitespaceCharSequenceNormalizer.java
create mode 100644
opennlp-core/opennlp-runtime/src/main/resources/opennlp/tools/tokenize/uax29/ExtendedPictographic.txt
create mode 100644
opennlp-core/opennlp-runtime/src/main/resources/opennlp/tools/tokenize/uax29/WordBreakProperty.txt
create mode 100644
opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/uax29/ExtendedPictographicTest.java
create mode 100644
opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/uax29/WordBoundaryConformanceTest.java
create mode 100644
opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/uax29/WordBreakPropertyTest.java
create mode 100644
opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/uax29/WordBreakTest.java
create mode 100644
opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/uax29/WordSegmenterTest.java
create mode 100644
opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/uax29/WordTokenizerTest.java
create mode 100644
opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/uax29/WordTypeTest.java
create mode 100644
opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/AlignedNormalizerPipelineTest.java
create mode 100644
opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/LineBreakPreservingWhitespaceCharSequenceNormalizerTest.java
create mode 100644
opennlp-core/opennlp-runtime/src/test/resources/opennlp/tools/tokenize/uax29/WordBreakTest.txt
