This is an automated email from the ASF dual-hosted git repository.
mawiesne pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/opennlp.git
The following commit(s) were added to refs/heads/main by this push:
new 011195f34 OPENNLP-1878: Non-breaking performance follow-up for
normalization and tokenization hot paths (#1161)
011195f34 is described below
commit 011195f3492d5b4e3845ee36e0ea34d7a51940c5
Author: Kristian Rickert <[email protected]>
AuthorDate: Sun Jul 19 06:40:27 2026 -0400
OPENNLP-1878: Non-breaking performance follow-up for normalization and
tokenization hot paths (#1161)
* OPENNLP-1878: Performance follow-up for normalization and tokenization
hot paths
Extract CodePoints for shared BMP-aware decoding, add member-free identity
short-circuits and Confusables fast paths, pre-size Alignment.Builder and
DL collections. Output-preserving; JMH shows up to ~4.3x on clean whitespace
folds and ~1.7x on Confusables skeleton for ASCII tokens.
New tests pin the previously unverified behavior: CodePointsTest exercises
at() and before() against Character.codePointAt/codePointBefore for BMP
text, a supplementary pair, and lone high and low surrogates, and an aligned
digit fold test asserts the span mapping over mixed ASCII and non-ASCII digits
so the equal-run treatment of ASCII digits stays pinned against Alignment
changes.
---
.../main/java/opennlp/tools/util/StringUtil.java | 8 +
.../opennlp/tools/util/normalizer/Alignment.java | 68 +++++-
.../opennlp/tools/util/normalizer/CharClass.java | 268 +++++++++++++--------
.../opennlp/tools/util/normalizer/CodePoints.java | 92 +++++++
.../tools/util/normalizer/AlignmentTest.java | 32 +++
.../tools/util/normalizer/CharClassTest.java | 91 ++++++-
.../tools/util/normalizer/CodePointsTest.java | 138 +++++++++++
.../src/main/java/opennlp/dl/AbstractDL.java | 4 +-
.../opennlp/dl/doccat/DocumentCategorizerDL.java | 12 +-
.../java/opennlp/dl/namefinder/NameFinderDL.java | 10 +-
.../AlignedAggregateCharSequenceNormalizer.java | 5 +-
.../opennlp/tools/util/normalizer/Confusables.java | 109 ++++++---
.../normalizer/DigitCharSequenceNormalizer.java | 15 +-
.../tools/tokenize/uax29/WordSegmenterTest.java | 7 +
.../normalizer/AlignedNormalizerPipelineTest.java | 17 ++
.../tools/util/normalizer/ConfusablesTest.java | 49 ++++
.../util/normalizer/SetBasedNormalizerTest.java | 7 +
17 files changed, 776 insertions(+), 156 deletions(-)
diff --git a/opennlp-api/src/main/java/opennlp/tools/util/StringUtil.java
b/opennlp-api/src/main/java/opennlp/tools/util/StringUtil.java
index 5de9ce6b5..98cca5989 100644
--- a/opennlp-api/src/main/java/opennlp/tools/util/StringUtil.java
+++ b/opennlp-api/src/main/java/opennlp/tools/util/StringUtil.java
@@ -30,6 +30,14 @@ public class StringUtil {
private static final Logger logger =
LoggerFactory.getLogger(StringUtil.class);
+ /**
+ * The ten ASCII digit strings {@code "0"} to {@code "9"}, indexed by digit
value. Precomputed so
+ * code folding digits to ASCII does not allocate a new single-character
string per digit; the
+ * list is immutable and safe to share.
+ */
+ public static final List<String> ASCII_DIGIT_STRINGS =
+ List.of("0", "1", "2", "3", "4", "5", "6", "7", "8", "9");
+
/**
* Determines if the specified {@link Character} is a whitespace under the
active
* {@link WhitespaceMode}: the Unicode {@code White_Space} property by
default, or, when
diff --git
a/opennlp-api/src/main/java/opennlp/tools/util/normalizer/Alignment.java
b/opennlp-api/src/main/java/opennlp/tools/util/normalizer/Alignment.java
index 6e1a1fc61..278becd86 100644
--- a/opennlp-api/src/main/java/opennlp/tools/util/normalizer/Alignment.java
+++ b/opennlp-api/src/main/java/opennlp/tools/util/normalizer/Alignment.java
@@ -161,8 +161,14 @@ public final class Alignment {
return new Alignment(starts, ends, originalLength);
}
- // First normalized index whose original coverage ends strictly after offset
(so it covers or
- // follows offset); normalizedLength() when offset is at or past the last
covered original char.
+ /**
+ * Finds the first normalized index whose original coverage ends strictly
after {@code offset}, so
+ * it covers or follows {@code offset}.
+ *
+ * @param offset An original offset.
+ * @return The first matching normalized index, or {@link
#normalizedLength()} when {@code offset}
+ * is at or past the last covered original character.
+ */
private int firstIndexEndingAfter(int offset) {
int low = 0;
int high = originalEnd.length;
@@ -177,7 +183,12 @@ public final class Alignment {
return low;
}
- // First normalized index whose original coverage starts at or after offset.
+ /**
+ * Finds the first normalized index whose original coverage starts at or
after {@code offset}.
+ *
+ * @param offset An original offset.
+ * @return The first matching normalized index.
+ */
private int firstIndexStartingAtOrAfter(int offset) {
int low = 0;
int high = originalStart.length;
@@ -192,6 +203,14 @@ public final class Alignment {
return low;
}
+ /**
+ * Validates that {@code [start, end)} is a half-open range within {@code
[0, length]}.
+ *
+ * @param start The inclusive start offset.
+ * @param end The exclusive end offset.
+ * @param length The length bounding the range.
+ * @throws IndexOutOfBoundsException Thrown if the offsets are out of range
or inverted.
+ */
private static void checkRange(int start, int end, int length) {
if (start < 0 || end > length || start > end) {
throw new IndexOutOfBoundsException("span [" + start + ", " + end + ")
is outside [0, "
@@ -207,12 +226,38 @@ public final class Alignment {
public static final class Builder {
private static final int MAX_ARRAY_SIZE = Integer.MAX_VALUE - 8;
+ private static final int DEFAULT_CAPACITY = 16;
- private int[] starts = new int[16];
- private int[] ends = new int[16];
+ private int[] starts;
+ private int[] ends;
private int count;
private int originalCursor;
+ /** Creates a builder with a small default capacity of {@link
#DEFAULT_CAPACITY} entries. */
+ public Builder() {
+ this(DEFAULT_CAPACITY);
+ }
+
+ /**
+ * Creates a builder pre-sized for a normalized text of about {@code
expectedLength}
+ * characters, so recording the edits of a typical pass (one entry per
normalized character)
+ * does not regrow the backing arrays. The value is a sizing hint only; it
does not limit how
+ * many edits can be recorded.
+ *
+ * @param expectedLength The expected normalized length, typically the
length of the text
+ * being normalized; must not be negative.
+ * @throws IllegalArgumentException Thrown if {@code expectedLength} is
negative.
+ */
+ public Builder(int expectedLength) {
+ if (expectedLength < 0) {
+ throw new IllegalArgumentException(
+ "The expectedLength must not be negative: " + expectedLength);
+ }
+ final int capacity = Math.max(DEFAULT_CAPACITY, expectedLength);
+ starts = new int[capacity];
+ ends = new int[capacity];
+ }
+
/**
* Records {@code charCount} characters copied through unchanged (a one to
one run).
*
@@ -272,6 +317,12 @@ public final class Alignment {
return new Alignment(Arrays.copyOf(starts, count), Arrays.copyOf(ends,
count), originalLength);
}
+ /**
+ * Appends one normalized character's original range, growing the backing
arrays as needed.
+ *
+ * @param start The inclusive original start offset.
+ * @param end The exclusive original end offset.
+ */
private void append(int start, int end) {
if (count == starts.length) {
grow();
@@ -281,8 +332,11 @@ public final class Alignment {
count++;
}
- // Overflow-aware 1.5x growth: never wraps to a negative capacity,
degrades to a clean
- // OutOfMemoryError at the array-size ceiling instead of
NegativeArraySizeException.
+ /**
+ * Grows the backing arrays by 1.5x, capped at the maximum array size.
+ *
+ * @throws OutOfMemoryError Thrown when the required capacity exceeds the
maximum array size.
+ */
private void grow() {
int newCapacity = starts.length + (starts.length >> 1);
if (newCapacity < 0 || newCapacity > MAX_ARRAY_SIZE) {
diff --git
a/opennlp-api/src/main/java/opennlp/tools/util/normalizer/CharClass.java
b/opennlp-api/src/main/java/opennlp/tools/util/normalizer/CharClass.java
index 5786220d1..7abae2a23 100644
--- a/opennlp-api/src/main/java/opennlp/tools/util/normalizer/CharClass.java
+++ b/opennlp-api/src/main/java/opennlp/tools/util/normalizer/CharClass.java
@@ -21,6 +21,7 @@ import java.util.List;
import java.util.function.IntFunction;
import opennlp.tools.util.Span;
+import opennlp.tools.util.normalizer.CodePoints.At;
/**
* A configurable class of Unicode code points and the cursor based operations
over it.
@@ -30,11 +31,8 @@ import opennlp.tools.util.Span;
* presets ({@link #whitespace()}, {@link #dashes()}); any other class is one
more configured
* instance with no new engine code.</p>
*
- * <p>Every operation is a single forward pass that reads one code point
- * ({@link Character#codePointAt(CharSequence, int)}), tests membership in
O(1), acts, and advances
- * by {@link Character#charCount(int)}. There is no regular expression, no
{@link java.util.regex}
- * allocation, and no reliance on {@link Character#isWhitespace(int)} or
- * {@link Character#isSpaceChar(int)}, all of which disagree with the Unicode
standard.</p>
+ * <p>Every operation is a single forward cursor pass over the text: no
regular expression and no
+ * per-call allocation beyond the result.</p>
*
* <p>Instances are immutable and thread-safe.</p>
*/
@@ -130,8 +128,8 @@ public final class CharClass {
int tokenStart = -1;
int i = 0;
while (i < length) {
- final int codePoint = Character.codePointAt(text, i);
- if (members.contains(codePoint)) {
+ final At cp = CodePoints.at(text, i);
+ if (members.contains(cp.codePoint())) {
if (tokenStart >= 0) {
spans.add(new Span(tokenStart, i));
tokenStart = -1;
@@ -139,7 +137,7 @@ public final class CharClass {
} else if (tokenStart < 0) {
tokenStart = i;
}
- i += Character.charCount(codePoint);
+ i = cp.nextIndex(i);
}
if (tokenStart >= 0) {
spans.add(new Span(tokenStart, length));
@@ -167,15 +165,22 @@ public final class CharClass {
/**
* Replaces each member code point with the replacement, one for one.
*
+ * <p>When no code point of {@code text} is a member, the text is returned
unchanged (as its
+ * {@link CharSequence#toString() string form}) without copying.</p>
+ *
* @param text The text to normalize.
* @return The normalized text.
* @throws IllegalArgumentException Thrown if {@code text} is {@code null}.
*/
public String normalize(CharSequence text) {
requireNonNullArg(text, "text");
- final StringBuilder out = new StringBuilder(text.length());
final int length = text.length();
- int i = 0;
+ final int first = firstMember(text);
+ if (first == length) {
+ return text.toString();
+ }
+ final StringBuilder out = new StringBuilder(length).append(text, 0, first);
+ int i = first;
while (i < length) {
final int codePoint = Character.codePointAt(text, i);
out.appendCodePoint(members.contains(codePoint) ? replacement :
codePoint);
@@ -192,15 +197,22 @@ public final class CharClass {
* the empty string). Use {@link #trim(CharSequence)} to drop edge members,
or collapse and then
* trim to do both.</p>
*
+ * <p>When no code point of {@code text} is a member, the text is returned
unchanged (as its
+ * {@link CharSequence#toString() string form}) without copying.</p>
+ *
* @param text The text to collapse.
* @return The collapsed text.
* @throws IllegalArgumentException Thrown if {@code text} is {@code null}.
*/
public String collapse(CharSequence text) {
requireNonNullArg(text, "text");
- final StringBuilder out = new StringBuilder(text.length());
final int length = text.length();
- int i = 0;
+ final int first = firstMember(text);
+ if (first == length) {
+ return text.toString();
+ }
+ final StringBuilder out = new StringBuilder(length).append(text, 0, first);
+ int i = first;
while (i < length) {
final int codePoint = Character.codePointAt(text, i);
if (members.contains(codePoint)) {
@@ -235,23 +247,23 @@ public final class CharClass {
final int length = text.length();
int i = 0;
while (i < length) {
- final int codePoint = Character.codePointAt(text, i);
- if (members.contains(codePoint)) {
- boolean preserve = keep.contains(codePoint);
- int j = i + Character.charCount(codePoint);
+ final At cp = CodePoints.at(text, i);
+ if (members.contains(cp.codePoint())) {
+ boolean preserve = keep.contains(cp.codePoint());
+ int j = cp.nextIndex(i);
while (j < length) {
- final int next = Character.codePointAt(text, j);
- if (!members.contains(next)) {
+ final At next = CodePoints.at(text, j);
+ if (!members.contains(next.codePoint())) {
break;
}
- preserve |= keep.contains(next);
- j += Character.charCount(next);
+ preserve |= keep.contains(next.codePoint());
+ j = next.nextIndex(j);
}
out.appendCodePoint(preserve ? keepReplacement : replacement);
i = j;
} else {
- out.appendCodePoint(codePoint);
- i += Character.charCount(codePoint);
+ out.appendCodePoint(cp.codePoint());
+ i = cp.nextIndex(i);
}
}
return out.toString();
@@ -269,19 +281,19 @@ public final class CharClass {
final int length = text.length();
int start = 0;
while (start < length) {
- final int codePoint = Character.codePointAt(text, start);
- if (!members.contains(codePoint)) {
+ final At cp = CodePoints.at(text, start);
+ if (!members.contains(cp.codePoint())) {
break;
}
- start += Character.charCount(codePoint);
+ start = cp.nextIndex(start);
}
int end = length;
while (end > start) {
- final int codePoint = Character.codePointBefore(text, end);
- if (!members.contains(codePoint)) {
+ final At cp = CodePoints.before(text, end);
+ if (!members.contains(cp.codePoint())) {
break;
}
- end -= Character.charCount(codePoint);
+ end = cp.previousIndex(end);
}
return text.subSequence(start, end).toString();
}
@@ -289,15 +301,22 @@ public final class CharClass {
/**
* Removes every member code point.
*
+ * <p>When no code point of {@code text} is a member, the text is returned
unchanged (as its
+ * {@link CharSequence#toString() string form}) without copying.</p>
+ *
* @param text The text to filter.
* @return The text with all members removed.
* @throws IllegalArgumentException Thrown if {@code text} is {@code null}.
*/
public String removeAll(CharSequence text) {
requireNonNullArg(text, "text");
- final StringBuilder out = new StringBuilder(text.length());
final int length = text.length();
- int i = 0;
+ final int first = firstMember(text);
+ if (first == length) {
+ return text.toString();
+ }
+ final StringBuilder out = new StringBuilder(length).append(text, 0, first);
+ int i = first;
while (i < length) {
final int codePoint = Character.codePointAt(text, i);
if (!members.contains(codePoint)) {
@@ -319,20 +338,19 @@ public final class CharClass {
public AlignedText normalizeAligned(CharSequence text) {
requireNonNullArg(text, "text");
final StringBuilder out = new StringBuilder(text.length());
- final Alignment.Builder alignment = new Alignment.Builder();
+ final Alignment.Builder alignment = new Alignment.Builder(text.length());
final int length = text.length();
int i = 0;
while (i < length) {
- final int codePoint = Character.codePointAt(text, i);
- final int charCount = Character.charCount(codePoint);
- if (members.contains(codePoint)) {
+ final At cp = CodePoints.at(text, i);
+ if (members.contains(cp.codePoint())) {
out.appendCodePoint(replacement);
- alignment.replace(charCount, Character.charCount(replacement));
+ alignment.replace(cp.charCount(), Character.charCount(replacement));
} else {
- out.appendCodePoint(codePoint);
- alignment.equal(charCount);
+ out.appendCodePoint(cp.codePoint());
+ alignment.equal(cp.charCount());
}
- i += charCount;
+ i = cp.nextIndex(i);
}
return new AlignedText(text, out.toString(), alignment.build(length));
}
@@ -348,21 +366,20 @@ public final class CharClass {
public AlignedText collapseAligned(CharSequence text) {
requireNonNullArg(text, "text");
final StringBuilder out = new StringBuilder(text.length());
- final Alignment.Builder alignment = new Alignment.Builder();
+ final Alignment.Builder alignment = new Alignment.Builder(text.length());
final int length = text.length();
int i = 0;
while (i < length) {
- final int codePoint = Character.codePointAt(text, i);
- if (members.contains(codePoint)) {
+ final At cp = CodePoints.at(text, i);
+ if (members.contains(cp.codePoint())) {
final int runEnd = skipRun(text, i);
out.appendCodePoint(replacement);
alignment.replace(runEnd - i, Character.charCount(replacement));
i = runEnd;
} else {
- final int charCount = Character.charCount(codePoint);
- out.appendCodePoint(codePoint);
- alignment.equal(charCount);
- i += charCount;
+ out.appendCodePoint(cp.codePoint());
+ alignment.equal(cp.charCount());
+ i = cp.nextIndex(i);
}
}
return new AlignedText(text, out.toString(), alignment.build(length));
@@ -385,31 +402,30 @@ public final class CharClass {
requireNonNullArg(keep, "keep");
requireValidCodePoint(keepReplacement);
final StringBuilder out = new StringBuilder(text.length());
- final Alignment.Builder alignment = new Alignment.Builder();
+ final Alignment.Builder alignment = new Alignment.Builder(text.length());
final int length = text.length();
int i = 0;
while (i < length) {
- final int codePoint = Character.codePointAt(text, i);
- if (members.contains(codePoint)) {
- boolean preserve = keep.contains(codePoint);
- int j = i + Character.charCount(codePoint);
+ final At cp = CodePoints.at(text, i);
+ if (members.contains(cp.codePoint())) {
+ boolean preserve = keep.contains(cp.codePoint());
+ int j = cp.nextIndex(i);
while (j < length) {
- final int next = Character.codePointAt(text, j);
- if (!members.contains(next)) {
+ final At next = CodePoints.at(text, j);
+ if (!members.contains(next.codePoint())) {
break;
}
- preserve |= keep.contains(next);
- j += Character.charCount(next);
+ preserve |= keep.contains(next.codePoint());
+ j = next.nextIndex(j);
}
final int emitted = preserve ? keepReplacement : replacement;
out.appendCodePoint(emitted);
alignment.replace(j - i, Character.charCount(emitted));
i = j;
} else {
- final int charCount = Character.charCount(codePoint);
- out.appendCodePoint(codePoint);
- alignment.equal(charCount);
- i += charCount;
+ out.appendCodePoint(cp.codePoint());
+ alignment.equal(cp.charCount());
+ i = cp.nextIndex(i);
}
}
return new AlignedText(text, out.toString(), alignment.build(length));
@@ -429,21 +445,21 @@ public final class CharClass {
final int length = text.length();
int start = 0;
while (start < length) {
- final int codePoint = Character.codePointAt(text, start);
- if (!members.contains(codePoint)) {
+ final At cp = CodePoints.at(text, start);
+ if (!members.contains(cp.codePoint())) {
break;
}
- start += Character.charCount(codePoint);
+ start = cp.nextIndex(start);
}
int end = length;
while (end > start) {
- final int codePoint = Character.codePointBefore(text, end);
- if (!members.contains(codePoint)) {
+ final At cp = CodePoints.before(text, end);
+ if (!members.contains(cp.codePoint())) {
break;
}
- end -= Character.charCount(codePoint);
+ end = cp.previousIndex(end);
}
- final Alignment.Builder alignment = new Alignment.Builder();
+ final Alignment.Builder alignment = new Alignment.Builder(text.length());
if (start > 0) {
alignment.replace(start, 0);
}
@@ -465,28 +481,30 @@ public final class CharClass {
public AlignedText removeAllAligned(CharSequence text) {
requireNonNullArg(text, "text");
final StringBuilder out = new StringBuilder(text.length());
- final Alignment.Builder alignment = new Alignment.Builder();
+ final Alignment.Builder alignment = new Alignment.Builder(text.length());
final int length = text.length();
int i = 0;
while (i < length) {
- final int codePoint = Character.codePointAt(text, i);
- final int charCount = Character.charCount(codePoint);
- if (members.contains(codePoint)) {
- alignment.replace(charCount, 0);
+ final At cp = CodePoints.at(text, i);
+ if (members.contains(cp.codePoint())) {
+ alignment.replace(cp.charCount(), 0);
} else {
- out.appendCodePoint(codePoint);
- alignment.equal(charCount);
+ out.appendCodePoint(cp.codePoint());
+ alignment.equal(cp.charCount());
}
- i += charCount;
+ i = cp.nextIndex(i);
}
return new AlignedText(text, out.toString(), alignment.build(length));
}
/**
* Applies a per-code-point substitution: each code point for which {@code
substitution} returns a
- * non-null string is replaced by that string, and the rest are copied
through. This is the shared,
- * offset-changing cursor pass behind the expanding folds (ellipsis, German
umlaut, digit), so each
- * of them supplies only a mapper rather than re-implementing the loop. No
regular expression.
+ * non-null string is replaced by that string, and the rest are copied
through. This is the shared
+ * cursor pass behind the expanding folds, with no regular expression.
+ *
+ * <p>When {@code substitution} returns {@code null} for every code point of
{@code text}, the
+ * text is returned unchanged (as its {@link CharSequence#toString() string
form}) without
+ * copying. The mapper is still applied exactly once per code point.</p>
*
* @param text The text to transform.
* @param substitution The replacement for a code point, or {@code null} to
copy it through.
@@ -496,18 +514,34 @@ public final class CharClass {
public static String substitute(CharSequence text, IntFunction<String>
substitution) {
requireNonNullArg(text, "text");
requireNonNullArg(substitution, "substitution");
- final StringBuilder out = new StringBuilder(text.length());
final int length = text.length();
- int i = 0;
+ String firstReplacement = null;
+ int first = 0;
+ int firstEnd = 0;
+ while (first < length) {
+ final At cp = CodePoints.at(text, first);
+ firstReplacement = substitution.apply(cp.codePoint());
+ if (firstReplacement != null) {
+ firstEnd = cp.nextIndex(first);
+ break;
+ }
+ first = cp.nextIndex(first);
+ }
+ if (firstReplacement == null) {
+ return text.toString();
+ }
+ final StringBuilder out =
+ new StringBuilder(length).append(text, 0,
first).append(firstReplacement);
+ int i = firstEnd;
while (i < length) {
- final int codePoint = Character.codePointAt(text, i);
- final String replacement = substitution.apply(codePoint);
+ final At cp = CodePoints.at(text, i);
+ final String replacement = substitution.apply(cp.codePoint());
if (replacement != null) {
out.append(replacement);
} else {
- out.appendCodePoint(codePoint);
+ out.appendCodePoint(cp.codePoint());
}
- i += Character.charCount(codePoint);
+ i = cp.nextIndex(i);
}
return out.toString();
}
@@ -525,48 +559,88 @@ public final class CharClass {
requireNonNullArg(text, "text");
requireNonNullArg(substitution, "substitution");
final StringBuilder out = new StringBuilder(text.length());
- final Alignment.Builder alignment = new Alignment.Builder();
+ final Alignment.Builder alignment = new Alignment.Builder(text.length());
final int length = text.length();
int i = 0;
while (i < length) {
- final int codePoint = Character.codePointAt(text, i);
- final int charCount = Character.charCount(codePoint);
- final String replacement = substitution.apply(codePoint);
+ final At cp = CodePoints.at(text, i);
+ final String replacement = substitution.apply(cp.codePoint());
if (replacement != null) {
out.append(replacement);
- alignment.replace(charCount, replacement.length());
+ alignment.replace(cp.charCount(), replacement.length());
} else {
- out.appendCodePoint(codePoint);
- alignment.equal(charCount);
+ out.appendCodePoint(cp.codePoint());
+ alignment.equal(cp.charCount());
}
- i += charCount;
+ i = cp.nextIndex(i);
}
return new AlignedText(text, out.toString(), alignment.build(length));
}
- // Returns the offset just past the maximal run of members starting at
runStart.
+ /**
+ * Finds the index of the first member code point, so callers can return
member-free text
+ * uncopied.
+ *
+ * @param text The text to scan.
+ * @return The index of the first member code point, or {@code
text.length()} when the text
+ * contains no member.
+ */
+ private int firstMember(CharSequence text) {
+ final int length = text.length();
+ int i = 0;
+ while (i < length) {
+ final At cp = CodePoints.at(text, i);
+ if (members.contains(cp.codePoint())) {
+ return i;
+ }
+ i = cp.nextIndex(i);
+ }
+ return length;
+ }
+
+ /**
+ * Advances past a run of member code points.
+ *
+ * @param text The text to scan.
+ * @param runStart The index where the member run starts.
+ * @return The index of the first non-member code point at or after {@code
runStart}, or
+ * {@code text.length()} when the run extends to the end of the text.
+ */
private int skipRun(CharSequence text, int runStart) {
final int length = text.length();
int i = runStart;
while (i < length) {
- final int codePoint = Character.codePointAt(text, i);
- if (!members.contains(codePoint)) {
+ final At cp = CodePoints.at(text, i);
+ if (!members.contains(cp.codePoint())) {
break;
}
- i += Character.charCount(codePoint);
+ i = cp.nextIndex(i);
}
return i;
}
+ /**
+ * Validates that {@code codePoint} is a Unicode code point.
+ *
+ * @param codePoint The value to validate.
+ * @throws IllegalArgumentException Thrown if {@code codePoint} is negative
or greater than
+ * {@link Character#MAX_CODE_POINT}.
+ */
private static void requireValidCodePoint(int codePoint) {
if (codePoint < 0 || codePoint > Character.MAX_CODE_POINT) {
throw new IllegalArgumentException("Not a Unicode code point: " +
codePoint);
}
}
- // Null parameters report IllegalArgumentException rather than
requireNonNull's
- // NullPointerException, so an invalid parameter and an invalid code point
surface through the
- // same exception type.
+ /**
+ * Validates that a parameter is not {@code null}.
+ *
+ * @param value The parameter value to validate.
+ * @param name The parameter name used in the error message.
+ * @param <T> The parameter type.
+ * @return {@code value}, never {@code null}.
+ * @throws IllegalArgumentException Thrown if {@code value} is {@code null}.
+ */
private static <T> T requireNonNullArg(T value, String name) {
if (value == null) {
throw new IllegalArgumentException("The " + name + " must not be null.");
diff --git
a/opennlp-api/src/main/java/opennlp/tools/util/normalizer/CodePoints.java
b/opennlp-api/src/main/java/opennlp/tools/util/normalizer/CodePoints.java
new file mode 100644
index 000000000..938d9d199
--- /dev/null
+++ b/opennlp-api/src/main/java/opennlp/tools/util/normalizer/CodePoints.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.tools.util.normalizer;
+
+/**
+ * Shared code-point decoding for forward (and reverse) cursor scans over
{@link CharSequence} text.
+ *
+ * <p>A {@code char} that is not a high surrogate is treated as its own code
point, so BMP text
+ * decodes without {@link Character#codePointAt(CharSequence, int)} on every
step.</p>
+ */
+final class CodePoints {
+
+ /**
+ * A code point at an index together with its UTF-16 width.
+ *
+ * <p>The advance methods pair with the decode direction: an {@code At} read
forward via
+ * {@link #at(CharSequence, int)} advances with {@link At#nextIndex(int)}
(taking the start
+ * index it was read at), and an {@code At} read backward via {@link
#before(CharSequence, int)}
+ * retreats with {@link At#previousIndex(int)} (taking the exclusive end
index it was read at).
+ * Mixing an {@code At} with the other direction's method yields a wrong
index.</p>
+ */
+ record At(int codePoint, int charCount) {
+
+ /**
+ * {@return the index immediately after this code point}
+ *
+ * @param index The start index this {@code At} was read at via {@link
#at(CharSequence, int)}.
+ */
+ int nextIndex(int index) {
+ return index + charCount;
+ }
+
+ /**
+ * {@return the index immediately before this code point when read from
the end}
+ *
+ * @param index The exclusive end index this {@code At} was read at via
+ * {@link #before(CharSequence, int)}.
+ */
+ int previousIndex(int index) {
+ return index - charCount;
+ }
+ }
+
+ private CodePoints() {
+ }
+
+ /**
+ * Reads the code point starting at {@code index}.
+ *
+ * @param text The text.
+ * @param index The UTF-16 index; must be in {@code [0, text.length())}.
+ * @return The decoded code point and its char width.
+ */
+ static At at(CharSequence text, int index) {
+ final char c = text.charAt(index);
+ if (!Character.isHighSurrogate(c)) {
+ return new At(c, 1);
+ }
+ final int codePoint = Character.codePointAt(text, index);
+ return new At(codePoint, Character.charCount(codePoint));
+ }
+
+ /**
+ * Reads the code point ending at {@code index} (exclusive end offset).
+ *
+ * @param text The text.
+ * @param index The UTF-16 index after the code point; must be in {@code (0,
text.length()]}.
+ * @return The decoded code point and its char width.
+ */
+ static At before(CharSequence text, int index) {
+ final char c = text.charAt(index - 1);
+ if (!Character.isLowSurrogate(c)) {
+ return new At(c, 1);
+ }
+ final int codePoint = Character.codePointBefore(text, index);
+ return new At(codePoint, Character.charCount(codePoint));
+ }
+}
diff --git
a/opennlp-api/src/test/java/opennlp/tools/util/normalizer/AlignmentTest.java
b/opennlp-api/src/test/java/opennlp/tools/util/normalizer/AlignmentTest.java
index 07c92de0f..27416f07f 100644
--- a/opennlp-api/src/test/java/opennlp/tools/util/normalizer/AlignmentTest.java
+++ b/opennlp-api/src/test/java/opennlp/tools/util/normalizer/AlignmentTest.java
@@ -156,6 +156,38 @@ public class AlignmentTest {
assertSpan(17, 18, a.toOriginalSpan(17, 18));
}
+ @Test
+ void testPreSizedBuilderMatchesDefaultBuilder() {
+ // The pre-size constructor is a capacity hint only: same edits, same
alignment.
+ final Alignment sized = new Alignment.Builder(4).equal(1).replace(2,
1).equal(1).build(4);
+ final Alignment grown = new Alignment.Builder().equal(1).replace(2,
1).equal(1).build(4);
+ assertEquals(grown.normalizedLength(), sized.normalizedLength());
+ assertEquals(grown.originalLength(), sized.originalLength());
+ for (int i = 0; i <= sized.normalizedLength(); i++) {
+ assertEquals(grown.toOriginalOffset(i), sized.toOriginalOffset(i));
+ }
+ assertSpan(1, 3, sized.toOriginalSpan(1, 2));
+ }
+
+ @Test
+ void testPreSizedBuilderIsAHintNotALimit() {
+ // Recording more edits than the hint must grow, not fail.
+ final Alignment a = new Alignment.Builder(2).equal(40).build(40);
+ assertEquals(40, a.normalizedLength());
+ assertSpan(0, 40, a.toOriginalSpan(0, 40));
+ }
+
+ @Test
+ void testPreSizedBuilderAcceptsZero() {
+ final Alignment a = new Alignment.Builder(0).equal(3).build(3);
+ assertEquals(3, a.normalizedLength());
+ }
+
+ @Test
+ void testPreSizedBuilderRejectsNegativeLength() {
+ assertThrows(IllegalArgumentException.class, () -> new
Alignment.Builder(-1));
+ }
+
@Test
void testAndThenChainsThreeStages() {
// "a b" -> "a b" (collapse) -> "a-b" (space->dash) -> "a_b"
(dash->underscore).
diff --git
a/opennlp-api/src/test/java/opennlp/tools/util/normalizer/CharClassTest.java
b/opennlp-api/src/test/java/opennlp/tools/util/normalizer/CharClassTest.java
index 5595be2be..29a92f1c6 100644
--- a/opennlp-api/src/test/java/opennlp/tools/util/normalizer/CharClassTest.java
+++ b/opennlp-api/src/test/java/opennlp/tools/util/normalizer/CharClassTest.java
@@ -17,6 +17,7 @@
package opennlp.tools.util.normalizer;
import java.util.List;
+import java.util.concurrent.atomic.AtomicInteger;
import org.junit.jupiter.api.Test;
@@ -26,6 +27,7 @@ import
opennlp.tools.util.normalizer.UnicodeWhitespace.WhitespaceCharacter;
import static org.junit.jupiter.api.Assertions.assertArrayEquals;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertSame;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
@@ -154,6 +156,92 @@ public class CharClassTest {
assertEquals("abcd", WS.removeAll("a b\tc d"));
}
+ // --- identity short-circuit (member-free input is returned uncopied)
----------------------
+
+ @Test
+ void testNormalizeReturnsSameInstanceWhenNoMember() {
+ final String text = "plain ascii token";
+ assertSame(text, DASH.normalize(text), "member-free input must be returned
uncopied");
+ final String noWhitespace = "token";
+ assertSame(noWhitespace, WS.normalize(noWhitespace));
+ }
+
+ @Test
+ void testCollapseReturnsSameInstanceWhenNoMember() {
+ final String text = "token";
+ assertSame(text, WS.collapse(text));
+ assertSame(text, DASH.collapse(text));
+ }
+
+ @Test
+ void testRemoveAllReturnsSameInstanceWhenNoMember() {
+ final String text = "token" + GRINNING_FACE;
+ assertSame(text, WS.removeAll(text));
+ assertSame(text, DASH.removeAll(text));
+ }
+
+ @Test
+ void testSubstituteReturnsSameInstanceWhenMapperNeverFires() {
+ final String text = "token 123";
+ assertSame(text, CharClass.substitute(text, codePoint -> null));
+ }
+
+ @Test
+ void testIdentityShortCircuitConvertsNonStringInput() {
+ final StringBuilder text = new StringBuilder("token");
+ assertEquals("token", WS.normalize(text), "a CharSequence input still
yields its string form");
+ assertEquals("token", WS.collapse(text));
+ assertEquals("token", WS.removeAll(text));
+ assertEquals("token", CharClass.substitute(text, codePoint -> null));
+ }
+
+ @Test
+ void testLazyBuilderCopiesUnchangedPrefixBeforeFirstMember() {
+ // The first member appears after a supplementary character, so the
pre-filled prefix must be
+ // copied by chars, not code points, to stay byte-identical.
+ final String text = GRINNING_FACE + "ab" + NBSP + NBSP + "cd";
+ assertEquals(GRINNING_FACE + "ab cd", WS.normalize(text));
+ assertEquals(GRINNING_FACE + "ab cd", WS.collapse(text));
+ assertEquals(GRINNING_FACE + "abcd", WS.removeAll(text));
+ assertEquals(GRINNING_FACE + "ab__cd",
+ CharClass.substitute(text, codePoint -> codePoint == 0x00A0 ? "_" :
null));
+ }
+
+ @Test
+ void testSubstituteAppliesMapperOncePerCodePoint() {
+ final String text = "a" + GRINNING_FACE + "1b2";
+ final AtomicInteger calls = new AtomicInteger();
+ final String unchanged = CharClass.substitute(text, codePoint -> {
+ calls.incrementAndGet();
+ return null;
+ });
+ assertSame(text, unchanged);
+ assertEquals(5, calls.get(), "the mapper must be applied exactly once per
code point");
+
+ calls.set(0);
+ final String replaced = CharClass.substitute(text, codePoint -> {
+ calls.incrementAndGet();
+ return Character.isDigit(codePoint) ? "#" : null;
+ });
+ assertEquals("a" + GRINNING_FACE + "#b#", replaced);
+ assertEquals(5, calls.get(), "the lazy builder must not re-apply the
mapper");
+ }
+
+ // --- BMP fast path: unpaired surrogates must decode exactly like
Character.codePointAt ----
+
+ @Test
+ void testUnpairedSurrogatesPassThroughUnchanged() {
+ final String loneHigh = "\uD83D"; // high surrogate with no low surrogate
following
+ final String loneLow = "\uDE00"; // low surrogate with no high surrogate
preceding
+ final String text = "a" + loneHigh + " " + loneLow + "b";
+ assertEquals(text, WS.normalize(text));
+ assertEquals(text, WS.collapse(text));
+ assertEquals("a" + loneHigh + loneLow + "b", WS.removeAll(text));
+ assertEquals(text, WS.trim(" " + text + "\t"));
+ assertArrayEquals(new String[] {"a" + loneHigh, loneLow + "b"},
WS.split(text));
+ assertEquals(text, CharClass.substitute(text, codePoint -> null));
+ }
+
// --- split / splitSpans
------------------------------------------------------------------
@Test
@@ -398,9 +486,6 @@ public class CharClassTest {
@Test
void nullParametersAreRejectedWithIllegalArgumentException() {
- // The whole class reports IllegalArgumentException for a null parameter
rather than
- // requireNonNull's NullPointerException, so an invalid parameter and an
invalid code point
- // surface through the same exception type (review).
final CharClass ws = CharClass.whitespace();
final CodePointSet nl = CodePointSet.of('\n');
assertThrows(IllegalArgumentException.class, () -> CharClass.of(null, '
'));
diff --git
a/opennlp-api/src/test/java/opennlp/tools/util/normalizer/CodePointsTest.java
b/opennlp-api/src/test/java/opennlp/tools/util/normalizer/CodePointsTest.java
new file mode 100644
index 000000000..889cd0c54
--- /dev/null
+++
b/opennlp-api/src/test/java/opennlp/tools/util/normalizer/CodePointsTest.java
@@ -0,0 +1,138 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.tools.util.normalizer;
+
+import org.junit.jupiter.api.Test;
+
+import opennlp.tools.util.normalizer.CodePoints.At;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+/**
+ * Exercises {@link CodePoints} directly: the decode of {@link
CodePoints#at(CharSequence, int)} and
+ * {@link CodePoints#before(CharSequence, int)} must agree with
+ * {@link Character#codePointAt(CharSequence, int)} and
+ * {@link Character#codePointBefore(CharSequence, int)} for BMP characters,
paired surrogates, and
+ * both kinds of unpaired surrogate.
+ */
+public class CodePointsTest {
+
+ private static final int GRINNING_FACE = 0x1F600; // a supplementary code
point (surrogate pair)
+ private static final char HIGH = '\uD83D'; // the high surrogate of
GRINNING_FACE
+ private static final char LOW = '\uDE00'; // the low surrogate of
GRINNING_FACE
+
+ @Test
+ void testAtDecodesBmpCharacter() {
+ final At cp = CodePoints.at("abc", 1);
+ assertEquals('b', cp.codePoint());
+ assertEquals(1, cp.charCount());
+ assertEquals(2, cp.nextIndex(1));
+ }
+
+ @Test
+ void testAtDecodesSupplementaryPair() {
+ final String text = "a" + new String(Character.toChars(GRINNING_FACE)) +
"b";
+ final At cp = CodePoints.at(text, 1);
+ assertEquals(GRINNING_FACE, cp.codePoint());
+ assertEquals(2, cp.charCount());
+ assertEquals(3, cp.nextIndex(1));
+ }
+
+ @Test
+ void testAtDecodesLoneHighSurrogateAsItself() {
+ // A high surrogate with no low surrogate after it is its own (unpaired)
code point.
+ final String text = "a" + HIGH + "b";
+ final At cp = CodePoints.at(text, 1);
+ assertEquals(HIGH, cp.codePoint());
+ assertEquals(1, cp.charCount());
+ assertEquals(Character.codePointAt(text, 1), cp.codePoint());
+ // A trailing high surrogate at the very end of the text must decode the
same way.
+ final String trailing = "a" + HIGH;
+ final At last = CodePoints.at(trailing, 1);
+ assertEquals(HIGH, last.codePoint());
+ assertEquals(1, last.charCount());
+ }
+
+ @Test
+ void testAtDecodesLoneLowSurrogateAsItself() {
+ final String text = "a" + LOW + "b";
+ final At cp = CodePoints.at(text, 1);
+ assertEquals(LOW, cp.codePoint());
+ assertEquals(1, cp.charCount());
+ assertEquals(Character.codePointAt(text, 1), cp.codePoint());
+ }
+
+ @Test
+ void testBeforeDecodesBmpCharacter() {
+ final At cp = CodePoints.before("abc", 2);
+ assertEquals('b', cp.codePoint());
+ assertEquals(1, cp.charCount());
+ assertEquals(1, cp.previousIndex(2));
+ }
+
+ @Test
+ void testBeforeDecodesSupplementaryPair() {
+ final String text = "a" + new String(Character.toChars(GRINNING_FACE)) +
"b";
+ final At cp = CodePoints.before(text, 3);
+ assertEquals(GRINNING_FACE, cp.codePoint());
+ assertEquals(2, cp.charCount());
+ assertEquals(1, cp.previousIndex(3));
+ }
+
+ @Test
+ void testBeforeDecodesLoneHighSurrogateAsItself() {
+ final String text = "a" + HIGH + "b";
+ final At cp = CodePoints.before(text, 2);
+ assertEquals(HIGH, cp.codePoint());
+ assertEquals(1, cp.charCount());
+ assertEquals(Character.codePointBefore(text, 2), cp.codePoint());
+ }
+
+ @Test
+ void testBeforeDecodesLoneLowSurrogateAsItself() {
+ // A low surrogate with no high surrogate before it is its own (unpaired)
code point.
+ final String text = "a" + LOW + "b";
+ final At cp = CodePoints.before(text, 2);
+ assertEquals(LOW, cp.codePoint());
+ assertEquals(1, cp.charCount());
+ assertEquals(Character.codePointBefore(text, 2), cp.codePoint());
+ // A leading low surrogate at the very start of the text must decode the
same way.
+ final String leading = LOW + "a";
+ final At first = CodePoints.before(leading, 1);
+ assertEquals(LOW, first.codePoint());
+ assertEquals(1, first.charCount());
+ }
+
+ @Test
+ void testEveryPositionAgreesWithCharacterCodePointAt() {
+ // Reference check over a text mixing BMP, a pair, and both unpaired
surrogate kinds.
+ final String text = "a" + new String(Character.toChars(GRINNING_FACE)) +
HIGH + "z" + LOW;
+ int i = 0;
+ while (i < text.length()) {
+ final At cp = CodePoints.at(text, i);
+ assertEquals(Character.codePointAt(text, i), cp.codePoint(), "at index "
+ i);
+ assertEquals(Character.charCount(Character.codePointAt(text, i)),
cp.charCount());
+ i = cp.nextIndex(i);
+ }
+ int end = text.length();
+ while (end > 0) {
+ final At cp = CodePoints.before(text, end);
+ assertEquals(Character.codePointBefore(text, end), cp.codePoint(),
"before index " + end);
+ end = cp.previousIndex(end);
+ }
+ }
+}
diff --git
a/opennlp-core/opennlp-ml/opennlp-dl/src/main/java/opennlp/dl/AbstractDL.java
b/opennlp-core/opennlp-ml/opennlp-dl/src/main/java/opennlp/dl/AbstractDL.java
index 5b61fc6e8..a59819927 100644
---
a/opennlp-core/opennlp-ml/opennlp-dl/src/main/java/opennlp/dl/AbstractDL.java
+++
b/opennlp-core/opennlp-ml/opennlp-dl/src/main/java/opennlp/dl/AbstractDL.java
@@ -415,8 +415,8 @@ public abstract class AbstractDL implements AutoCloseable {
// An AlignedText whose alignment is the identity, for the case where no
length-changing fold was
// applied so the folded text has the same length and offsets as the
original.
private static AlignedText identityAligned(final String original, final
String normalized) {
- final Alignment alignment =
- new
Alignment.Builder().equal(normalized.length()).build(normalized.length());
+ final Alignment alignment = new Alignment.Builder(normalized.length())
+ .equal(normalized.length()).build(normalized.length());
return new AlignedText(original, normalized, alignment);
}
diff --git
a/opennlp-core/opennlp-ml/opennlp-dl/src/main/java/opennlp/dl/doccat/DocumentCategorizerDL.java
b/opennlp-core/opennlp-ml/opennlp-dl/src/main/java/opennlp/dl/doccat/DocumentCategorizerDL.java
index a9c6dd9c5..e5fbe2c6a 100644
---
a/opennlp-core/opennlp-ml/opennlp-dl/src/main/java/opennlp/dl/doccat/DocumentCategorizerDL.java
+++
b/opennlp-core/opennlp-ml/opennlp-dl/src/main/java/opennlp/dl/doccat/DocumentCategorizerDL.java
@@ -22,10 +22,10 @@ import java.io.IOException;
import java.nio.LongBuffer;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
-import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -214,7 +214,7 @@ public class DocumentCategorizerDL extends AbstractDL
implements DocumentCategor
"The document to categorize must contain at least one non-whitespace
token");
}
- final List<double[]> scores = new LinkedList<>();
+ final List<double[]> scores = new ArrayList<>(tokens.size());
for (final Tokens t : tokens) {
scores.add(softmax(infer(t)));
}
@@ -240,7 +240,8 @@ public class DocumentCategorizerDL extends AbstractDL
implements DocumentCategor
*/
private float[] infer(final Tokens t) {
- final Map<String, OnnxTensor> inputs = new HashMap<>();
+ // At most three inputs (ids, attention mask, token type ids), so size for
exactly that.
+ final Map<String, OnnxTensor> inputs = HashMap.newHashMap(3);
final Object output;
try {
inputs.put(INPUT_IDS, OnnxTensor.createTensor(env,
@@ -367,12 +368,13 @@ public class DocumentCategorizerDL extends AbstractDL
implements DocumentCategor
private List<Tokens> tokenize(final String input) {
final String text = normalizeInput(input, normalizeWhitespace,
normalizeDashes);
- final List<Tokens> t = new LinkedList<>();
// Segment long input text into overlapping chunks (split on Unicode
whitespace) configured by
// InferenceOptions before feeding each chunk into BERT.
//
https://medium.com/analytics-vidhya/text-classification-with-bert-using-transformers-for-long-text-inputs-f54833994dfd
- for (final String group : whitespaceChunks(text, documentSplitSize,
splitOverlapSize)) {
+ final List<String> groups = whitespaceChunks(text, documentSplitSize,
splitOverlapSize);
+ final List<Tokens> t = new ArrayList<>(groups.size());
+ for (final String group : groups) {
// Now we can tokenize the group and continue.
final String[] tokens = tokenizer.tokenize(group);
diff --git
a/opennlp-core/opennlp-ml/opennlp-dl/src/main/java/opennlp/dl/namefinder/NameFinderDL.java
b/opennlp-core/opennlp-ml/opennlp-dl/src/main/java/opennlp/dl/namefinder/NameFinderDL.java
index 2da72e58d..95bb81d87 100644
---
a/opennlp-core/opennlp-ml/opennlp-dl/src/main/java/opennlp/dl/namefinder/NameFinderDL.java
+++
b/opennlp-core/opennlp-ml/opennlp-dl/src/main/java/opennlp/dl/namefinder/NameFinderDL.java
@@ -24,7 +24,6 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.Comparator;
import java.util.HashMap;
-import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -352,7 +351,8 @@ public class NameFinderDL extends AbstractDL implements
OffsetMappingNameFinder
*/
private float[][] infer(final Tokens tokens) {
- final Map<String, OnnxTensor> inputs = new HashMap<>();
+ // At most three inputs (ids, attention mask, token type ids), so size for
exactly that.
+ final Map<String, OnnxTensor> inputs = HashMap.newHashMap(3);
final Object output;
try {
inputs.put(INPUT_IDS, OnnxTensor.createTensor(env,
LongBuffer.wrap(tokens.ids()),
@@ -781,13 +781,13 @@ public class NameFinderDL extends AbstractDL implements
OffsetMappingNameFinder
private List<ChunkTokens> tokenize(final String text) {
- final List<ChunkTokens> t = new LinkedList<>();
-
// Segment long input text into overlapping chunks (split on Unicode
whitespace) configured by
// InferenceOptions before feeding each chunk into BERT, keeping each
chunk's character span so
// its decoded spans can be bounded to the region the chunk covers.
//
https://medium.com/analytics-vidhya/text-classification-with-bert-using-transformers-for-long-text-inputs-f54833994dfd
- for (final TextChunk chunk : whitespaceChunkSpans(text, documentSplitSize,
splitOverlapSize)) {
+ final List<TextChunk> chunks = whitespaceChunkSpans(text,
documentSplitSize, splitOverlapSize);
+ final List<ChunkTokens> t = new ArrayList<>(chunks.size());
+ for (final TextChunk chunk : chunks) {
// Now we can tokenize the group and continue.
final String[] tokens = tokenizer.tokenize(chunk.text());
diff --git
a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/AlignedAggregateCharSequenceNormalizer.java
b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/AlignedAggregateCharSequenceNormalizer.java
index c13592e73..efa318f47 100644
---
a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/AlignedAggregateCharSequenceNormalizer.java
+++
b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/AlignedAggregateCharSequenceNormalizer.java
@@ -34,6 +34,7 @@ final class AlignedAggregateCharSequenceNormalizer implements
OffsetAwareNormali
this.steps = steps;
}
+ /** {@inheritDoc} */
@Override
public CharSequence normalize(CharSequence text) {
CharSequence result = text;
@@ -43,6 +44,7 @@ final class AlignedAggregateCharSequenceNormalizer implements
OffsetAwareNormali
return result;
}
+ /** {@inheritDoc} */
@Override
public AlignedText normalizeAligned(CharSequence text) {
if (steps.length == 0) {
@@ -50,7 +52,8 @@ final class AlignedAggregateCharSequenceNormalizer implements
OffsetAwareNormali
// from the stored original for a CharSequence whose length() differs
from its toString().
final String identity = text.toString();
return new AlignedText(identity, identity,
- new
Alignment.Builder().equal(identity.length()).build(identity.length()));
+ new Alignment.Builder(identity.length()).equal(identity.length())
+ .build(identity.length()));
}
// Normalize the input to a String once so the stored original and the
per-stage alignment
// lengths agree even for a CharSequence whose length() differs from its
toString().
diff --git
a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/Confusables.java
b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/Confusables.java
index 6fda61c54..4e252761a 100644
---
a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/Confusables.java
+++
b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/Confusables.java
@@ -23,6 +23,7 @@ import java.io.InputStreamReader;
import java.io.UncheckedIOException;
import java.nio.charset.StandardCharsets;
import java.text.Normalizer;
+import java.util.BitSet;
import java.util.HashMap;
import java.util.Map;
@@ -34,53 +35,60 @@ import java.util.Map;
* lookalikes, exactly when their skeletons are equal.
*
* <p>The mapping is loaded once from the {@code confusables.txt} resource of
the Unicode security
- * data (parsed with simple cursor scanning, no regular expression). The
skeleton of a string is
- * {@code NFD(map(NFD(s)))}: decompose, replace each code point with its
prototype, and decompose
- * again. This changes length and offsets, so it belongs to the derived,
matching-only form rather
- * than to any offset-preserving transform.</p>
+ * data. The skeleton of a string is {@code NFD(map(NFD(s)))}: decompose,
replace each code point
+ * with its prototype, and decompose again. This changes length and offsets,
so it is a
+ * matching-only comparison form, not an offset-preserving transform.</p>
*
- * <p>This implements only the skeleton transform and the confusable-detection
test built on
- * skeleton equality. The other mechanisms defined in UTS #39, such as
identifier
+ * <p>This implements only the UTS #39 skeleton transform and the
confusable-detection test
+ * built on skeleton equality. The other mechanisms defined in the report,
such as identifier
* restriction levels, mixed-script and whole-script confusable detection, and
the bidirectional
- * skeleton, are out of scope; the skeleton here is a comparison form, not a
security-grade
- * conformance claim for the full report.</p>
+ * skeleton, are out of scope.</p>
*/
public final class Confusables {
private static final String RESOURCE = "confusables.txt";
- // Maps a single confusable code point to its prototype sequence (one or
more code points).
- private static volatile Map<Integer, String> prototypes;
-
- private Confusables() {
+ private record Data(Map<Integer, String> prototypes, BitSet keys) {
}
- private static Map<Integer, String> prototypes() {
- Map<Integer, String> map = prototypes;
- if (map == null) {
- synchronized (Confusables.class) {
- map = prototypes;
- if (map == null) {
- map = load();
- prototypes = map;
- }
- }
- }
- return map;
+ private static final Data DATA = load();
+
+ private Confusables() {
}
- private static Map<Integer, String> load() {
+ /**
+ * {@return the prototype mapping and key set parsed from the bundled {@code
confusables.txt}
+ * resource}
+ *
+ * @throws IllegalStateException Thrown if the resource is missing.
+ * @throws UncheckedIOException Thrown if the resource cannot be read.
+ */
+ private static Data load() {
try (InputStream in = Confusables.class.getResourceAsStream(RESOURCE)) {
if (in == null) {
throw new IllegalStateException("Missing confusables data resource: "
+ RESOURCE);
}
- return parse(in);
+ final Map<Integer, String> prototypes = parse(in);
+ final BitSet keys = new BitSet();
+ for (final int codePoint : prototypes.keySet()) {
+ keys.set(codePoint);
+ }
+ return new Data(prototypes, keys);
} catch (IOException e) {
throw new UncheckedIOException("Unable to read confusables data resource
" + RESOURCE, e);
}
}
- // Package-private so the malformed-data handling can be exercised without
the bundled resource.
+ /**
+ * Parses the source-to-prototype mappings from the {@code confusables.txt}
data. Package-private
+ * so the malformed-data handling can be exercised without the bundled
resource.
+ *
+ * @param in The data stream.
+ * @return The source code point to prototype string mapping.
+ * @throws IOException Thrown if the stream cannot be read.
+ * @throws IllegalArgumentException Thrown if a line is structurally
malformed or holds a
+ * non-hexadecimal code point.
+ */
static Map<Integer, String> parse(InputStream in) throws IOException {
final Map<Integer, String> map = new HashMap<>();
try (BufferedReader reader =
@@ -106,8 +114,7 @@ public final class Confusables {
final int source = Integer.parseInt(content.substring(0,
firstSemicolon).strip(), 16);
final String target = content.substring(firstSemicolon + 1,
secondSemicolon).strip();
final StringBuilder prototype = new StringBuilder();
- // Scan the whitespace-delimited hex tokens by hand to honor the
no-regex contract and
- // avoid compiling a Pattern for every one of the ~10k lines during
static init.
+ // Scan the whitespace-delimited hex tokens by hand, with no regular
expression.
final int targetLength = target.length();
int pos = 0;
while (pos < targetLength) {
@@ -140,15 +147,42 @@ public final class Confusables {
*
* @param text The text to reduce.
* @return The skeleton.
+ * @throws IllegalArgumentException Thrown if {@code text} is {@code null}.
*/
public static String skeleton(CharSequence text) {
- final Map<Integer, String> map = prototypes();
+ if (text == null) {
+ throw new IllegalArgumentException("text must not be null");
+ }
+ final Data d = DATA;
+ final BitSet keys = d.keys();
+
+ // Clean ASCII or NFD text with no confusable keys skips both Normalizer
passes.
+ final int length = text.length();
+ boolean asciiOnly = true;
+ boolean anyKey = false;
+ int i = 0;
+ while (i < length) {
+ final int codePoint = Character.codePointAt(text, i);
+ if (codePoint >= 0x80) {
+ asciiOnly = false;
+ }
+ if (keys.get(codePoint)) {
+ anyKey = true;
+ break;
+ }
+ i += Character.charCount(codePoint);
+ }
+ if (!anyKey && (asciiOnly || Normalizer.isNormalized(text,
Normalizer.Form.NFD))) {
+ return text.toString();
+ }
+
+ final Map<Integer, String> map = d.prototypes();
final String decomposed = Normalizer.normalize(text, Normalizer.Form.NFD);
final StringBuilder mapped = new StringBuilder(decomposed.length());
- for (int i = 0; i < decomposed.length(); ) {
- final int codePoint = decomposed.codePointAt(i);
- i += Character.charCount(codePoint);
- final String prototype = map.get(codePoint);
+ for (int j = 0; j < decomposed.length(); ) {
+ final int codePoint = decomposed.codePointAt(j);
+ j += Character.charCount(codePoint);
+ final String prototype = keys.get(codePoint) ? map.get(codePoint) : null;
if (prototype != null) {
mapped.append(prototype);
} else {
@@ -164,8 +198,15 @@ public final class Confusables {
*
* @param left The first string.
* @param right The second string.
+ * @throws IllegalArgumentException Thrown if {@code left} or {@code right}
is {@code null}.
*/
public static boolean confusable(CharSequence left, CharSequence right) {
+ if (left == null) {
+ throw new IllegalArgumentException("left must not be null");
+ }
+ if (right == null) {
+ throw new IllegalArgumentException("right must not be null");
+ }
return skeleton(left).equals(skeleton(right));
}
}
diff --git
a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/DigitCharSequenceNormalizer.java
b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/DigitCharSequenceNormalizer.java
index 0d80b09ae..6735dcdf0 100644
---
a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/DigitCharSequenceNormalizer.java
+++
b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/DigitCharSequenceNormalizer.java
@@ -16,6 +16,8 @@
*/
package opennlp.tools.util.normalizer;
+import opennlp.tools.util.StringUtil;
+
/**
* A {@link CharSequenceNormalizer} that maps Unicode decimal digits to their
ASCII equivalents,
* so for example Arabic-Indic, Devanagari, or fullwidth digits all become
{@code 0}-{@code 9}.
@@ -37,17 +39,26 @@ public class DigitCharSequenceNormalizer implements
OffsetAwareNormalizer {
return INSTANCE;
}
+ /** {@inheritDoc} */
@Override
public CharSequence normalize(CharSequence text) {
return CharClass.substitute(text, DigitCharSequenceNormalizer::toAscii);
}
- // The ASCII digit for a Unicode decimal digit code point, or null to copy
the code point through.
+ /**
+ * Maps a code point to its ASCII digit fold.
+ *
+ * @param codePoint The code point to fold.
+ * @return The ASCII digit string for a non-ASCII Unicode decimal digit, or
{@code null} to copy
+ * the code point through (ASCII digits are already their own fold).
+ */
private static String toAscii(int codePoint) {
final int value = Character.digit(codePoint, 10);
- return value >= 0 ? String.valueOf((char) ('0' + value)) : null;
+ return value >= 0 && codePoint != '0' + value
+ ? StringUtil.ASCII_DIGIT_STRINGS.get(value) : null;
}
+ /** {@inheritDoc} */
@Override
public AlignedText normalizeAligned(CharSequence text) {
return CharClass.substituteAligned(text,
DigitCharSequenceNormalizer::toAscii);
diff --git
a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/uax29/WordSegmenterTest.java
b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/uax29/WordSegmenterTest.java
index 96dbc1491..6e120b7be 100644
---
a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/uax29/WordSegmenterTest.java
+++
b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/uax29/WordSegmenterTest.java
@@ -97,6 +97,13 @@ public class WordSegmenterTest {
assertEquals(List.of(flag), words(flag)); // WB15/WB16
}
+ @Test
+ void testUnpairedSurrogatesAdvanceOneCharAndSegmentAsOther() {
+ // A lone high or low surrogate must decode exactly like
Character.codePointAt: one char,
+ // Word_Break class Other, so it separates from the letters on both sides.
+ assertEquals(List.of("ab", "\uD800", "cd", "\uDC00", "ef"),
words("ab\uD800cd\uDC00ef"));
+ }
+
@Test
void testEmptyText() {
assertEquals(List.of(), words(""));
diff --git
a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/AlignedNormalizerPipelineTest.java
b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/AlignedNormalizerPipelineTest.java
index c6327f47b..07098dfbc 100644
---
a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/AlignedNormalizerPipelineTest.java
+++
b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/AlignedNormalizerPipelineTest.java
@@ -318,6 +318,23 @@ public class AlignedNormalizerPipelineTest {
assertEquals(cp(MATH_BOLD_DIGIT_ZERO), covered(aligned, 1, 2));
}
+ @Test
+ void digitFoldOfMixedAsciiAndNonAsciiDigitsMapsSpansBackToOriginal() {
+ // ASCII digits take the copy-through path (recorded as equal runs),
non-ASCII digits are
+ // replaced; the resulting alignment must be indistinguishable from a full
replace pass.
+ final String original = "4" + cp(0x0665) + "2" + cp(0xFF11); // '4',
arabic-indic 5, '2', fullwidth 1
+ final AlignedText aligned = DigitCharSequenceNormalizer.getInstance()
+ .normalizeAligned(original);
+ assertEquals("4521", aligned.normalized());
+ // Every position folds one for one, so each normalized digit maps back to
its own source.
+ assertEquals("4", covered(aligned, 0, 1));
+ assertEquals(cp(0x0665), covered(aligned, 1, 2));
+ assertEquals("2", covered(aligned, 2, 3));
+ assertEquals(cp(0xFF11), covered(aligned, 3, 4));
+ // And the reverse direction: the ASCII '2' at original index 2 maps to
normalized index 2.
+ assertEquals(new Span(2, 3), aligned.toNormalizedSpan(2, 3));
+ }
+
@Test
void quoteFoldMapsSpanBackToOriginal() {
final String original = cp(0x201C) + "hi" + cp(0x201D); // curly double
quotes
diff --git
a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/ConfusablesTest.java
b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/ConfusablesTest.java
index 262fe5aa9..3a1491495 100644
---
a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/ConfusablesTest.java
+++
b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/ConfusablesTest.java
@@ -20,6 +20,8 @@ import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertSame;
+import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
public class ConfusablesTest {
@@ -72,10 +74,57 @@ public class ConfusablesTest {
assertEquals(Confusables.skeleton("data"), Confusables.skeleton(spoof));
}
+ @Test
+ void testSkeletonReturnsCleanAsciiUnchanged() {
+ // No code point of the text is a prototype key, so the fast path returns
the input as-is.
+ final String clean = "hello";
+ assertSame(clean, Confusables.skeleton(clean));
+ }
+
+ @Test
+ void testSkeletonStillMapsAsciiPrototypeKeys() {
+ // ASCII code points that ARE prototype keys must defeat the fast path and
map as before.
+ assertEquals("rn", Confusables.skeleton("m"));
+ assertEquals("l", Confusables.skeleton("I"));
+ assertEquals("l", Confusables.skeleton("1"));
+ assertEquals("O", Confusables.skeleton("0"));
+ assertEquals("corn", Confusables.skeleton("com"));
+ }
+
+ @Test
+ void testSkeletonStillDecomposesPrecomposedText() {
+ // A precomposed character is not in NFD form, so it must take the full
path and decompose,
+ // and the result must agree with the already-decomposed spelling
(canonical equivalence).
+ final String precomposed = cp(0x00E9); // e with acute, single
code point
+ final String decomposed = "e" + cp(0x0301); // e + combining acute
+ assertEquals(decomposed, Confusables.skeleton(precomposed));
+ assertEquals(Confusables.skeleton(precomposed),
Confusables.skeleton(decomposed));
+ }
+
+ @Test
+ void testSkeletonReturnsNfdStableNonKeyTextUnchanged() {
+ // Non-ASCII, already in NFD form, and no prototype key: the fast path
applies.
+ final String hiragana = cp(0x3042);
+ assertSame(hiragana, Confusables.skeleton(hiragana));
+ }
+
+ @Test
+ void testSkeletonStillMapsNonAsciiPrototypeKeys() {
+ // U+4E00 is a prototype key (maps to the prolonged sound mark), so it
must still map.
+ assertEquals(cp(0x30FC), Confusables.skeleton(cp(0x4E00)));
+ }
+
@Test
void testTermConfusableFoldDimension() {
final String spoof = "p" + cp(0x0430) + "yp" + cp(0x0430) + "l";
final TermAnalyzer analyzer =
TermAnalyzer.builder().confusableFold().build();
assertEquals(Confusables.skeleton("paypal"),
analyzer.analyze(spoof).get(0).normalized());
}
+
+ @Test
+ void testNullArgumentsAreRejected() {
+ assertThrows(IllegalArgumentException.class, () ->
Confusables.skeleton(null));
+ assertThrows(IllegalArgumentException.class, () ->
Confusables.confusable(null, "a"));
+ assertThrows(IllegalArgumentException.class, () ->
Confusables.confusable("a", null));
+ }
}
diff --git
a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/SetBasedNormalizerTest.java
b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/SetBasedNormalizerTest.java
index ea333f06b..0197370aa 100644
---
a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/SetBasedNormalizerTest.java
+++
b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/SetBasedNormalizerTest.java
@@ -96,6 +96,13 @@ public class SetBasedNormalizerTest {
assertSame(DigitCharSequenceNormalizer.getInstance(),
DigitCharSequenceNormalizer.getInstance());
}
+ @Test
+ void testDigitsReturnAsciiDigitTextUncopied() {
+ // ASCII digits are already their own fold, so digit-clean text
short-circuits uncopied.
+ final String text = "version 42 of 2026";
+ assertSame(text,
DigitCharSequenceNormalizer.getInstance().normalize(text));
+ }
+
// --- invisible / bidi controls
-----------------------------------------------------------
@Test
