This is an automated email from the ASF dual-hosted git repository.
mawiesne pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/opennlp.git
The following commit(s) were added to refs/heads/main by this push:
new db7adf0a6 OPENNLP-1876: Replace regex with cursor scans in the legacy
CharSequenceNormalizers (#1151)
db7adf0a6 is described below
commit db7adf0a6aa92a5524d058ec1dc7c0e4c0bf61e3
Author: Kristian Rickert <[email protected]>
AuthorDate: Thu Jul 16 15:20:33 2026 -0400
OPENNLP-1876: Replace regex with cursor scans in the legacy
CharSequenceNormalizers (#1151)
OPENNLP-1876: Replace regex with cursor scans in the legacy
CharSequenceNormalizers
* Add characterization tests for NumberCharSequenceNormalizer
* Replace the regex in NumberCharSequenceNormalizer with a cursor scan
* Add characterization tests for ShrinkCharSequenceNormalizer
* Replace the regexes in ShrinkCharSequenceNormalizer with cursor scans
* Add characterization tests for TwitterCharSequenceNormalizer
* Replace the regexes in TwitterCharSequenceNormalizer with cursor scans
* Add characterization tests for UrlCharSequenceNormalizer
* Replace the regexes in UrlCharSequenceNormalizer with cursor scans
* Add characterization tests for the spellcheck token guards
* Convert the spellcheck token guards to cursor scans
* Split on the literal delimiter without regex in SpellCorrectingTokenStream
---
.../util/normalizer/CharSequenceNormalizer.java | 2 +-
.../tools/langdetect/LanguageDetectorFactory.java | 15 +-
.../AccentFoldCharSequenceNormalizer.java | 6 +-
.../AggregateCharSequenceNormalizer.java | 4 +
.../opennlp/tools/util/normalizer/AsciiChars.java | 77 ++++++
.../normalizer/CaseFoldCharSequenceNormalizer.java | 6 +-
.../ConfusableSkeletonCharSequenceNormalizer.java | 6 +-
.../normalizer/EmojiCharSequenceNormalizer.java | 4 +
.../util/normalizer/NfcCharSequenceNormalizer.java | 6 +-
.../normalizer/NfkcCharSequenceNormalizer.java | 6 +-
.../normalizer/NumberCharSequenceNormalizer.java | 33 ++-
.../normalizer/ShrinkCharSequenceNormalizer.java | 128 ++++++++-
.../SocialMediaCharSequenceNormalizer.java | 306 +++++++++++++++++++++
.../normalizer/TwitterCharSequenceNormalizer.java | 55 ----
.../util/normalizer/UrlCharSequenceNormalizer.java | 192 ++++++++++++-
.../CharSequenceNormalizerContractTest.java | 88 ++++++
.../util/normalizer/CharacterizationInputs.java | 51 ++++
...CharSequenceNormalizerCharacterizationTest.java | 102 +++++++
...CharSequenceNormalizerCharacterizationTest.java | 166 +++++++++++
...CharSequenceNormalizerCharacterizationTest.java | 198 +++++++++++++
...CharSequenceNormalizerCharacterizationTest.java | 204 ++++++++++++++
.../SpellCheckingCharSequenceNormalizer.java | 192 +++++++++----
.../stream/SpellCorrectingObjectStream.java | 31 ++-
.../stream/SpellCorrectingTokenStream.java | 67 +++--
.../SpellCheckingCharSequenceNormalizerTest.java | 74 ++++-
.../stream/SpellCorrectingObjectStreamTest.java | 16 +-
.../stream/SpellCorrectingTokenStreamTest.java | 47 ++++
.../TwitterCharSequenceNormalizerTest.java | 61 ----
28 files changed, 1901 insertions(+), 242 deletions(-)
diff --git
a/opennlp-api/src/main/java/opennlp/tools/util/normalizer/CharSequenceNormalizer.java
b/opennlp-api/src/main/java/opennlp/tools/util/normalizer/CharSequenceNormalizer.java
index be813064d..2823c8124 100644
---
a/opennlp-api/src/main/java/opennlp/tools/util/normalizer/CharSequenceNormalizer.java
+++
b/opennlp-api/src/main/java/opennlp/tools/util/normalizer/CharSequenceNormalizer.java
@@ -29,7 +29,7 @@ public interface CharSequenceNormalizer extends Serializable {
/**
* Normalizes a sequence of characters.
*
- * @param text The {@link CharSequence} to normalize.
+ * @param text The {@link CharSequence} to normalize. Must not be {@code
null}.
* @return The normalized {@link CharSequence}.
* @throws IllegalArgumentException Thrown if {@code text} is {@code null}.
*/
diff --git
a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/langdetect/LanguageDetectorFactory.java
b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/langdetect/LanguageDetectorFactory.java
index 3796f2bea..873643177 100644
---
a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/langdetect/LanguageDetectorFactory.java
+++
b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/langdetect/LanguageDetectorFactory.java
@@ -23,7 +23,7 @@ import opennlp.tools.util.ext.ExtensionLoader;
import opennlp.tools.util.normalizer.EmojiCharSequenceNormalizer;
import opennlp.tools.util.normalizer.NumberCharSequenceNormalizer;
import opennlp.tools.util.normalizer.ShrinkCharSequenceNormalizer;
-import opennlp.tools.util.normalizer.TwitterCharSequenceNormalizer;
+import opennlp.tools.util.normalizer.SocialMediaCharSequenceNormalizer;
import opennlp.tools.util.normalizer.UrlCharSequenceNormalizer;
@@ -33,14 +33,9 @@ import
opennlp.tools.util.normalizer.UrlCharSequenceNormalizer;
* Extend this class to change the Language Detector behaviour,
* such as the {@link LanguageDetectorContextGenerator}.
* The default {@link DefaultLanguageDetectorContextGenerator} will use char
n-grams of
- * size 1 to 3 and the following normalizers:
- * <ul>
- * <li> {@link EmojiCharSequenceNormalizer}
- * <li> {@link UrlCharSequenceNormalizer}
- * <li> {@link TwitterCharSequenceNormalizer}
- * <li> {@link NumberCharSequenceNormalizer}
- * <li> {@link ShrinkCharSequenceNormalizer}
- * </ul>
+ * size 1 to 3 and the {@link EmojiCharSequenceNormalizer}, {@link
UrlCharSequenceNormalizer},
+ * {@link SocialMediaCharSequenceNormalizer}, {@link
NumberCharSequenceNormalizer}, and
+ * {@link ShrinkCharSequenceNormalizer}.
*/
public class LanguageDetectorFactory extends BaseToolFactory {
@@ -53,7 +48,7 @@ public class LanguageDetectorFactory extends BaseToolFactory {
return new DefaultLanguageDetectorContextGenerator(1, 3,
EmojiCharSequenceNormalizer.getInstance(),
UrlCharSequenceNormalizer.getInstance(),
- TwitterCharSequenceNormalizer.getInstance(),
+ SocialMediaCharSequenceNormalizer.getInstance(),
NumberCharSequenceNormalizer.getInstance(),
ShrinkCharSequenceNormalizer.getInstance());
}
diff --git
a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/AccentFoldCharSequenceNormalizer.java
b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/AccentFoldCharSequenceNormalizer.java
index 76536306d..bc3a2534b 100644
---
a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/AccentFoldCharSequenceNormalizer.java
+++
b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/AccentFoldCharSequenceNormalizer.java
@@ -41,7 +41,7 @@ import java.util.Set;
*/
public class AccentFoldCharSequenceNormalizer implements
CharSequenceNormalizer {
- private static final long serialVersionUID = 7843116209554120071L;
+ private static final long serialVersionUID = -7502792584669454623L;
private static final Set<Character.UnicodeScript> DEFAULT_SCRIPTS = Set.of(
Character.UnicodeScript.LATIN,
@@ -74,8 +74,12 @@ public class AccentFoldCharSequenceNormalizer implements
CharSequenceNormalizer
return INSTANCE;
}
+ /** {@inheritDoc} */
@Override
public CharSequence normalize(CharSequence text) {
+ if (text == null) {
+ throw new IllegalArgumentException("The text must not be null.");
+ }
final String decomposed = Normalizer.normalize(text, Normalizer.Form.NFD);
final StringBuilder out = new StringBuilder(decomposed.length());
diff --git
a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/AggregateCharSequenceNormalizer.java
b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/AggregateCharSequenceNormalizer.java
index 429118c51..56a46b556 100644
---
a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/AggregateCharSequenceNormalizer.java
+++
b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/AggregateCharSequenceNormalizer.java
@@ -31,8 +31,12 @@ public class AggregateCharSequenceNormalizer implements
CharSequenceNormalizer {
this.normalizers = normalizers;
}
+ /** {@inheritDoc} */
@Override
public CharSequence normalize (CharSequence text) {
+ if (text == null) {
+ throw new IllegalArgumentException("The text must not be null.");
+ }
for (CharSequenceNormalizer normalizers : normalizers) {
text = normalizers.normalize(text);
diff --git
a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/AsciiChars.java
b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/AsciiChars.java
new file mode 100644
index 000000000..54dbb19e4
--- /dev/null
+++
b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/AsciiChars.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.tools.util.normalizer;
+
+/**
+ * ASCII character helpers shared by the normalizer implementations of this
package, so the
+ * character definitions cannot drift apart between classes.
+ */
+final class AsciiChars {
+
+ /**
+ * The six ASCII whitespace characters: tab ({@code U+0009}), line feed
({@code U+000A}),
+ * vertical tab ({@code U+000B}), form feed ({@code U+000C}), carriage return
+ * ({@code U+000D}), and space ({@code U+0020}).
+ */
+ static final CodePointSet WHITESPACE =
+ CodePointSet.ofRange(0x0009, 0x000D).union(CodePointSet.of(0x0020));
+
+ private AsciiChars() {
+ }
+
+ /**
+ * {@return {@code c} lower cased if it is an ASCII capital letter,
otherwise {@code c}
+ * unchanged}
+ *
+ * @param c The char to lower case.
+ */
+ static char toLower(char c) {
+ return c >= 'A' && c <= 'Z' ? (char) (c + 0x20) : c;
+ }
+
+ /**
+ * {@return {@code codePoint} lower cased if it is an ASCII capital letter,
otherwise
+ * {@code codePoint} unchanged}
+ *
+ * @param codePoint The code point to lower case.
+ */
+ static int toLower(int codePoint) {
+ return codePoint >= 'A' && codePoint <= 'Z' ? codePoint + 0x20 : codePoint;
+ }
+
+ /**
+ * {@return whether {@code a} and {@code b} are equal, comparing ASCII
letters
+ * case-insensitively}
+ *
+ * @param a The first char.
+ * @param b The second char.
+ */
+ static boolean caseInsensitiveEquals(char a, char b) {
+ return toLower(a) == toLower(b);
+ }
+
+ /**
+ * {@return whether {@code a} and {@code b} are equal, comparing ASCII
letters
+ * case-insensitively}
+ *
+ * @param a The first code point.
+ * @param b The second code point.
+ */
+ static boolean caseInsensitiveEquals(int a, int b) {
+ return toLower(a) == toLower(b);
+ }
+}
diff --git
a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/CaseFoldCharSequenceNormalizer.java
b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/CaseFoldCharSequenceNormalizer.java
index 02cf3e071..471aedc21 100644
---
a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/CaseFoldCharSequenceNormalizer.java
+++
b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/CaseFoldCharSequenceNormalizer.java
@@ -31,7 +31,7 @@ import java.util.Objects;
*/
public class CaseFoldCharSequenceNormalizer implements CharSequenceNormalizer {
- private static final long serialVersionUID = 9183472265510038829L;
+ private static final long serialVersionUID = -3462240068381722106L;
private static final CaseFoldCharSequenceNormalizer INSTANCE =
new CaseFoldCharSequenceNormalizer();
@@ -73,8 +73,12 @@ public class CaseFoldCharSequenceNormalizer implements
CharSequenceNormalizer {
return Locale.ROOT.equals(locale) ? INSTANCE : new
CaseFoldCharSequenceNormalizer(locale);
}
+ /** {@inheritDoc} */
@Override
public CharSequence normalize(CharSequence text) {
+ if (text == null) {
+ throw new IllegalArgumentException("The text must not be null.");
+ }
return text.toString().toLowerCase(locale);
}
}
diff --git
a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/ConfusableSkeletonCharSequenceNormalizer.java
b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/ConfusableSkeletonCharSequenceNormalizer.java
index 63e71c810..9b4e065bd 100644
---
a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/ConfusableSkeletonCharSequenceNormalizer.java
+++
b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/ConfusableSkeletonCharSequenceNormalizer.java
@@ -27,7 +27,7 @@ package opennlp.tools.util.normalizer;
*/
public class ConfusableSkeletonCharSequenceNormalizer implements
CharSequenceNormalizer {
- private static final long serialVersionUID = 3074516628190437751L;
+ private static final long serialVersionUID = 1068969405606517368L;
private static final ConfusableSkeletonCharSequenceNormalizer INSTANCE =
new ConfusableSkeletonCharSequenceNormalizer();
@@ -40,8 +40,12 @@ public class ConfusableSkeletonCharSequenceNormalizer
implements CharSequenceNor
return INSTANCE;
}
+ /** {@inheritDoc} */
@Override
public CharSequence normalize(CharSequence text) {
+ if (text == null) {
+ throw new IllegalArgumentException("The text must not be null.");
+ }
return Confusables.skeleton(text);
}
}
diff --git
a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/EmojiCharSequenceNormalizer.java
b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/EmojiCharSequenceNormalizer.java
index 867d6b264..d2b75335a 100644
---
a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/EmojiCharSequenceNormalizer.java
+++
b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/EmojiCharSequenceNormalizer.java
@@ -39,8 +39,12 @@ public class EmojiCharSequenceNormalizer implements
CharSequenceNormalizer {
private static final Pattern EMOJI_REGEX =
Pattern.compile("[\\uD83C-\\uDBFF\\uDC00-\\uDFFF]+");
+ /** {@inheritDoc} */
@Override
public CharSequence normalize (CharSequence text) {
+ if (text == null) {
+ throw new IllegalArgumentException("The text must not be null.");
+ }
return EMOJI_REGEX.matcher(text).replaceAll(" ");
}
}
diff --git
a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/NfcCharSequenceNormalizer.java
b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/NfcCharSequenceNormalizer.java
index 3b03a4a51..d16479aab 100644
---
a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/NfcCharSequenceNormalizer.java
+++
b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/NfcCharSequenceNormalizer.java
@@ -29,7 +29,7 @@ import java.text.Normalizer;
*/
public class NfcCharSequenceNormalizer implements CharSequenceNormalizer {
- private static final long serialVersionUID = 5960421783304915649L;
+ private static final long serialVersionUID = -2229543110136546341L;
private static final NfcCharSequenceNormalizer INSTANCE = new
NfcCharSequenceNormalizer();
@@ -38,8 +38,12 @@ public class NfcCharSequenceNormalizer implements
CharSequenceNormalizer {
return INSTANCE;
}
+ /** {@inheritDoc} */
@Override
public CharSequence normalize(CharSequence text) {
+ if (text == null) {
+ throw new IllegalArgumentException("The text must not be null.");
+ }
return Normalizer.normalize(text, Normalizer.Form.NFC);
}
}
diff --git
a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/NfkcCharSequenceNormalizer.java
b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/NfkcCharSequenceNormalizer.java
index c4e38997d..415928765 100644
---
a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/NfkcCharSequenceNormalizer.java
+++
b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/NfkcCharSequenceNormalizer.java
@@ -30,7 +30,7 @@ import java.text.Normalizer;
*/
public class NfkcCharSequenceNormalizer implements CharSequenceNormalizer {
- private static final long serialVersionUID = 8273650194427108359L;
+ private static final long serialVersionUID = -167565018805144793L;
private static final NfkcCharSequenceNormalizer INSTANCE = new
NfkcCharSequenceNormalizer();
@@ -39,8 +39,12 @@ public class NfkcCharSequenceNormalizer implements
CharSequenceNormalizer {
return INSTANCE;
}
+ /** {@inheritDoc} */
@Override
public CharSequence normalize(CharSequence text) {
+ if (text == null) {
+ throw new IllegalArgumentException("The text must not be null.");
+ }
return Normalizer.normalize(text, Normalizer.Form.NFKC);
}
}
diff --git
a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/NumberCharSequenceNormalizer.java
b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/NumberCharSequenceNormalizer.java
index d7e8ddf8b..23935606a 100644
---
a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/NumberCharSequenceNormalizer.java
+++
b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/NumberCharSequenceNormalizer.java
@@ -16,26 +16,43 @@
*/
package opennlp.tools.util.normalizer;
-import java.util.regex.Pattern;
-
/**
- * A {@link NumberCharSequenceNormalizer} implementation that normalizes text
- * in terms of numbers. Every encounter will be replaced by a whitespace.
+ * A {@link CharSequenceNormalizer} implementation that normalizes text in
terms of numbers:
+ * every maximal run of ASCII digits ({@code 0} to {@code 9}), that is the
longest unbroken
+ * stretch of consecutive digits, is replaced by a single space. For example,
{@code "a1234b56"}
+ * becomes {@code "a b "}. Non-ASCII digits, for example Arabic-Indic or
fullwidth digits, are
+ * not treated as digits and are left unchanged.
*/
public class NumberCharSequenceNormalizer implements CharSequenceNormalizer {
private static final long serialVersionUID = -782056416383201122L;
-
- private static final Pattern NUMBER_REGEX = Pattern.compile("\\d+");
+
+ private static final CharClass ASCII_DIGITS =
+ CharClass.of(CodePointSet.ofRange('0', '9'), ' ');
private static final NumberCharSequenceNormalizer INSTANCE = new
NumberCharSequenceNormalizer();
+ /** {@return the shared, stateless instance} */
public static NumberCharSequenceNormalizer getInstance() {
return INSTANCE;
}
+ /**
+ * {@inheritDoc}
+ */
@Override
- public CharSequence normalize (CharSequence text) {
- return NUMBER_REGEX.matcher(text).replaceAll(" ");
+ public CharSequence normalize(CharSequence text) {
+ if (text == null) {
+ throw new IllegalArgumentException("The text must not be null.");
+ }
+ // The common digit-free text is returned without copying, like the
sibling normalizers.
+ final int length = text.length();
+ for (int i = 0; i < length; i++) {
+ final char c = text.charAt(i);
+ if (c >= '0' && c <= '9') {
+ return ASCII_DIGITS.collapse(text);
+ }
+ }
+ return text;
}
}
diff --git
a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/ShrinkCharSequenceNormalizer.java
b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/ShrinkCharSequenceNormalizer.java
index 24e52de0d..20cbf7f9a 100644
---
a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/ShrinkCharSequenceNormalizer.java
+++
b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/ShrinkCharSequenceNormalizer.java
@@ -16,28 +16,136 @@
*/
package opennlp.tools.util.normalizer;
-import java.util.regex.Pattern;
-
/**
- * A {@link ShrinkCharSequenceNormalizer} implementation that shrinks repeated
spaces / chars in text.
+ * A {@link CharSequenceNormalizer} implementation that shrinks repeated
whitespace and repeated
+ * characters in text, in three steps:
+ * <ol>
+ * <li>Each run of two or more ASCII whitespace characters (tab, line feed,
vertical tab,
+ * form feed, carriage return, or space) collapses to a single space. A
lone whitespace
+ * character is kept as it is.</li>
+ * <li>Each run of three or more repeats of one code point shrinks to two
copies of its first
+ * occurrence, comparing repeats case-insensitively over ASCII; a run
never starts on a
+ * line terminator. For example, {@code "coooool"} becomes {@code
"cool"}.</li>
+ * <li>Every leading and trailing character at or below {@code U+0020} is
dropped, the same
+ * rule {@link String#trim()} applies.</li>
+ * </ol>
*/
public class ShrinkCharSequenceNormalizer implements CharSequenceNormalizer {
private static final long serialVersionUID = -4511969661556543048L;
- private static final Pattern REPEATED_CHAR_REGEX =
Pattern.compile("(.)\\1{2,}",
- Pattern.CASE_INSENSITIVE);
- private static final Pattern SPACE_REGEX = Pattern.compile("\\s{2,}",
- Pattern.CASE_INSENSITIVE);
+ /** The line terminator code points; a repeat run never starts on one. */
+ private static final CodePointSet LINE_TERMINATORS =
+ CodePointSet.of(0x000A, 0x000D, 0x0085, 0x2028, 0x2029);
private static final ShrinkCharSequenceNormalizer INSTANCE = new
ShrinkCharSequenceNormalizer();
+ /** {@return the shared, stateless instance} */
public static ShrinkCharSequenceNormalizer getInstance() {
return INSTANCE;
}
- public CharSequence normalize (CharSequence text) {
- text = SPACE_REGEX.matcher(text).replaceAll(" ");
- return REPEATED_CHAR_REGEX.matcher(text).replaceAll("$1$1").trim();
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public CharSequence normalize(CharSequence text) {
+ if (text == null) {
+ throw new IllegalArgumentException("The text must not be null.");
+ }
+ final CharSequence shrunk =
shrinkRepeatedCodePoints(shrinkWhitespace(text));
+ // Drops every leading and trailing char at or below U+0020, without
copying when nothing
+ // changed anywhere in the pipeline.
+ int start = 0;
+ int end = shrunk.length();
+ while (start < end && shrunk.charAt(start) <= ' ') {
+ start++;
+ }
+ while (end > start && shrunk.charAt(end - 1) <= ' ') {
+ end--;
+ }
+ if (start == 0 && end == shrunk.length()) {
+ return shrunk;
+ }
+ return shrunk.subSequence(start, end).toString();
+ }
+
+ /**
+ * Replaces each run of two or more ASCII whitespace characters with a
single space; a lone
+ * whitespace character stays as it is.
+ *
+ * @param text The text to scan; never null.
+ * @return The input itself when no run was found, otherwise the shrunk copy.
+ */
+ private CharSequence shrinkWhitespace(CharSequence text) {
+ final int length = text.length();
+ StringBuilder out = null;
+ int i = 0;
+ while (i < length) {
+ final int codePoint = Character.codePointAt(text, i);
+ if (AsciiChars.WHITESPACE.contains(codePoint)) {
+ int runEnd = i + 1; // members are single-char ASCII
+ while (runEnd < length &&
AsciiChars.WHITESPACE.contains(text.charAt(runEnd))) {
+ runEnd++;
+ }
+ if (runEnd - i >= 2) {
+ if (out == null) {
+ out = new StringBuilder(length).append(text, 0, i);
+ }
+ out.append(' ');
+ } else if (out != null) {
+ out.append(text.charAt(i));
+ }
+ i = runEnd;
+ } else {
+ if (out != null) {
+ out.appendCodePoint(codePoint);
+ }
+ i += Character.charCount(codePoint);
+ }
+ }
+ return out == null ? text : out.toString();
}
+
+ /**
+ * Shrinks each run of three or more repeats of one code point to two copies
of its first
+ * occurrence. The run's first code point must not be a line terminator,
every repeat
+ * advances by the first occurrence's char count while comparing whole code
points ASCII
+ * case-insensitively, and a failed attempt resumes at the next char.
+ *
+ * @param text The text to scan; never null.
+ * @return The input itself when no run was found, otherwise the shrunk copy.
+ */
+ private CharSequence shrinkRepeatedCodePoints(CharSequence text) {
+ final int length = text.length();
+ StringBuilder out = null;
+ int i = 0;
+ while (i < length) {
+ final int codePoint = Character.codePointAt(text, i);
+ if (!LINE_TERMINATORS.contains(codePoint)) {
+ final int charCount = Character.charCount(codePoint);
+ int repeats = 0;
+ int end = i + charCount;
+ while (end + charCount <= length
+ && AsciiChars.caseInsensitiveEquals(Character.codePointAt(text,
end), codePoint)) {
+ end += charCount;
+ repeats++;
+ }
+ if (repeats >= 2) {
+ if (out == null) {
+ out = new StringBuilder(length).append(text, 0, i);
+ }
+ out.append(text, i, i + charCount).append(text, i, i + charCount);
+ i = end;
+ continue;
+ }
+ }
+ if (out != null) {
+ out.append(text.charAt(i));
+ }
+ i++;
+ }
+ return out == null ? text : out.toString();
+ }
+
}
diff --git
a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/SocialMediaCharSequenceNormalizer.java
b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/SocialMediaCharSequenceNormalizer.java
new file mode 100644
index 000000000..fd6223935
--- /dev/null
+++
b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/SocialMediaCharSequenceNormalizer.java
@@ -0,0 +1,306 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.tools.util.normalizer;
+
+/**
+ * A {@link CharSequenceNormalizer} implementation that normalizes social
media text: hashtags,
+ * user handles, retweet markers, emoticons, and repeated laughter. Every
encounter will be
+ * replaced by a whitespace.
+ *
+ * <p>Normalization runs in four passes: a {@code #} or {@code @} together
with the following
+ * non-whitespace run becomes one space; each sequence of case-insensitive
{@code rt} units,
+ * each followed by a space or colon, becomes one space when it starts on a
word boundary; each
+ * emoticon (eyes {@code :}, {@code ;} or {@code x}, an optional {@code -}
nose, and a mouth out
+ * of {@code ( ) d o p}, case-insensitive) becomes one space, including inside
words. Finally,
+ * repeated laughter shrinks to two of its repeating units ({@code "hahaha"}
to {@code "haha"}),
+ * comparing ASCII case-insensitively.</p>
+ */
+public class SocialMediaCharSequenceNormalizer implements
CharSequenceNormalizer {
+
+ private static final long serialVersionUID = -5441528948820855349L;
+
+ private static final SocialMediaCharSequenceNormalizer INSTANCE = new
SocialMediaCharSequenceNormalizer();
+
+ /** {@return the shared, stateless instance} */
+ public static SocialMediaCharSequenceNormalizer getInstance() {
+ return INSTANCE;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public CharSequence normalize(CharSequence text) {
+ if (text == null) {
+ throw new IllegalArgumentException("The text must not be null.");
+ }
+ return
shrinkLaughter(removeEmoticons(removeRetweetMarkers(removeTagsAndHandles(text))));
+ }
+
+ /**
+ * Replaces each hashtag or handle with one space: a {@code #} or {@code @}
matches only if a
+ * non-whitespace char follows, and the match then swallows the whole
non-whitespace run,
+ * including further {@code #} and {@code @}.
+ *
+ * @param text The text to scan; never null.
+ * @return The input itself when nothing matched, otherwise the normalized
copy.
+ */
+ private CharSequence removeTagsAndHandles(CharSequence text) {
+ final int length = text.length();
+ StringBuilder out = null;
+ int i = 0;
+ while (i < length) {
+ final char c = text.charAt(i);
+ if ((c == '#' || c == '@')
+ && i + 1 < length && !AsciiChars.WHITESPACE.contains(text.charAt(i +
1))) {
+ int end = i + 2;
+ while (end < length &&
!AsciiChars.WHITESPACE.contains(text.charAt(end))) {
+ end++;
+ }
+ if (out == null) {
+ out = new StringBuilder(length).append(text, 0, i);
+ }
+ out.append(' ');
+ i = end;
+ } else {
+ if (out != null) {
+ out.append(c);
+ }
+ i++;
+ }
+ }
+ return out == null ? text : out.toString();
+ }
+
+ /**
+ * Replaces each sequence of one or more retweet units with one space when
the sequence
+ * starts on a word boundary.
+ *
+ * @param text The text to scan; never null.
+ * @return The input itself when nothing matched, otherwise the normalized
copy.
+ */
+ private CharSequence removeRetweetMarkers(CharSequence text) {
+ final int length = text.length();
+ StringBuilder out = null;
+ int i = 0;
+ while (i < length) {
+ final char c = text.charAt(i);
+ if ((c == 'r' || c == 'R') && !wordBeforeBlocksBoundary(text, i)) {
+ int end = i;
+ while (end + 3 <= length && isRetweetUnit(text, end)) {
+ end += 3;
+ }
+ if (end > i) {
+ if (out == null) {
+ out = new StringBuilder(length).append(text, 0, i);
+ }
+ out.append(' ');
+ i = end;
+ continue;
+ }
+ }
+ if (out != null) {
+ out.append(c);
+ }
+ i++;
+ }
+ return out == null ? text : out.toString();
+ }
+
+ /**
+ * {@return whether the three chars starting at {@code at} form one retweet
unit:
+ * {@code r} or {@code R}, {@code t} or {@code T}, then a space or colon}
+ *
+ * @param text The text to look into; never null.
+ * @param at The index of the unit's first char; at least three chars must
remain.
+ */
+ private boolean isRetweetUnit(CharSequence text, int at) {
+ final char r = text.charAt(at);
+ final char t = text.charAt(at + 1);
+ final char separator = text.charAt(at + 2);
+ return (r == 'r' || r == 'R') && (t == 't' || t == 'T')
+ && (separator == ' ' || separator == ':');
+ }
+
+ /**
+ * {@return whether the word boundary before {@code index} is blocked} The
boundary is
+ * blocked if the preceding code point is an ASCII word character, or is a
non-spacing mark
+ * whose backwards base-character scan reaches a letter or digit. The scan
reads one char at
+ * a time and stops on the low surrogate of a supplementary-plane mark; that
stop is part of
+ * the boundary behavior this normalizer preserves.
+ *
+ * @param text The text to look into; never null.
+ * @param index The index the boundary is checked before.
+ */
+ private boolean wordBeforeBlocksBoundary(CharSequence text, int index) {
+ if (index <= 0) {
+ return false;
+ }
+ final int before = Character.codePointBefore(text, index);
+ if (isAsciiWord(before)) {
+ return true;
+ }
+ if (Character.getType(before) == Character.NON_SPACING_MARK) {
+ for (int x = index - 1; x >= 0; x--) {
+ final int codePoint = Character.codePointAt(text, x);
+ if (Character.isLetterOrDigit(codePoint)) {
+ return true;
+ }
+ if (Character.getType(codePoint) != Character.NON_SPACING_MARK) {
+ return false;
+ }
+ }
+ }
+ return false;
+ }
+
+ /** {@return whether {@code codePoint} is an ASCII letter, digit, or
underscore} */
+ private boolean isAsciiWord(int codePoint) {
+ return codePoint >= 'a' && codePoint <= 'z'
+ || codePoint >= 'A' && codePoint <= 'Z'
+ || codePoint >= '0' && codePoint <= '9'
+ || codePoint == '_';
+ }
+
+ /**
+ * Replaces each emoticon with one space, including inside words: eyes, then
an optional
+ * {@code -} nose (tried first when present), then a mouth. A bare {@code -}
is never a
+ * mouth.
+ *
+ * @param text The text to scan; never null.
+ * @return The input itself when nothing matched, otherwise the normalized
copy.
+ */
+ private CharSequence removeEmoticons(CharSequence text) {
+ final int length = text.length();
+ StringBuilder out = null;
+ int i = 0;
+ while (i < length) {
+ final char c = text.charAt(i);
+ if (isEmoticonEyes(c)) {
+ if (i + 2 < length && text.charAt(i + 1) == '-' &&
isEmoticonMouth(text.charAt(i + 2))) {
+ if (out == null) {
+ out = new StringBuilder(length).append(text, 0, i);
+ }
+ out.append(' ');
+ i += 3;
+ continue;
+ }
+ if (i + 1 < length && isEmoticonMouth(text.charAt(i + 1))) {
+ if (out == null) {
+ out = new StringBuilder(length).append(text, 0, i);
+ }
+ out.append(' ');
+ i += 2;
+ continue;
+ }
+ }
+ if (out != null) {
+ out.append(c);
+ }
+ i++;
+ }
+ return out == null ? text : out.toString();
+ }
+
+ /** {@return whether {@code c} is an emoticon's eyes: {@code :}, {@code ;},
or {@code x}} */
+ private boolean isEmoticonEyes(char c) {
+ return c == ':' || c == ';' || c == 'x' || c == 'X';
+ }
+
+ /** {@return whether {@code c} is an emoticon's mouth: one of {@code ( ) d o
p}, either case} */
+ private boolean isEmoticonMouth(char c) {
+ return c == '(' || c == ')' || c == 'd' || c == 'D'
+ || c == 'o' || c == 'O' || c == 'p' || c == 'P';
+ }
+
+ /**
+ * Shrinks laughter: a run of {@code h}/{@code j}, a run of vowels, and at
least one complete
+ * repetition of runs of the two last characters of those runs, shrink to
those two
+ * characters twice. The character sets involved are pairwise disjoint, so
the greedy runs
+ * never overlap ambiguously; a failed attempt resumes at the next char.
+ *
+ * @param text The text to scan; never null.
+ * @return The input itself when nothing matched, otherwise the normalized
copy.
+ */
+ private CharSequence shrinkLaughter(CharSequence text) {
+ final int length = text.length();
+ StringBuilder out = null;
+ int i = 0;
+ while (i < length) {
+ final char c = text.charAt(i);
+ if (isLaughConsonant(c)) {
+ int p = i + 1;
+ while (p < length && isLaughConsonant(text.charAt(p))) {
+ p++;
+ }
+ if (p < length && isLaughVowel(text.charAt(p))) {
+ final char consonant = text.charAt(p - 1);
+ int q = p + 1;
+ while (q < length && isLaughVowel(text.charAt(q))) {
+ q++;
+ }
+ final char vowel = text.charAt(q - 1);
+ int end = q;
+ while (true) {
+ int consonantRunEnd = end;
+ while (consonantRunEnd < length
+ &&
AsciiChars.caseInsensitiveEquals(text.charAt(consonantRunEnd), consonant)) {
+ consonantRunEnd++;
+ }
+ if (consonantRunEnd == end) {
+ break;
+ }
+ int vowelRunEnd = consonantRunEnd;
+ while (vowelRunEnd < length
+ && AsciiChars.caseInsensitiveEquals(text.charAt(vowelRunEnd),
vowel)) {
+ vowelRunEnd++;
+ }
+ if (vowelRunEnd == consonantRunEnd) {
+ break;
+ }
+ end = vowelRunEnd;
+ }
+ if (end > q) {
+ if (out == null) {
+ out = new StringBuilder(length).append(text, 0, i);
+ }
+
out.append(consonant).append(vowel).append(consonant).append(vowel);
+ i = end;
+ continue;
+ }
+ }
+ }
+ if (out != null) {
+ out.append(c);
+ }
+ i++;
+ }
+ return out == null ? text : out.toString();
+ }
+
+ /** {@return whether {@code c} is a laughter consonant: {@code h} or {@code
j}, either case} */
+ private boolean isLaughConsonant(char c) {
+ return c == 'h' || c == 'H' || c == 'j' || c == 'J';
+ }
+
+ /** {@return whether {@code c} is a laughter vowel: {@code a e i o u},
either case} */
+ private boolean isLaughVowel(char c) {
+ return c == 'a' || c == 'i' || c == 'e' || c == 'o' || c == 'u'
+ || c == 'A' || c == 'I' || c == 'E' || c == 'O' || c == 'U';
+ }
+
+}
diff --git
a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/TwitterCharSequenceNormalizer.java
b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/TwitterCharSequenceNormalizer.java
deleted file mode 100644
index daeb3c974..000000000
---
a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/TwitterCharSequenceNormalizer.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package opennlp.tools.util.normalizer;
-
-import java.util.regex.Pattern;
-
-/**
- * A {@link TwitterCharSequenceNormalizer} implementation that normalizes text
- * in terms of Twitter character patterns. Every encounter will be replaced by
a whitespace.
- */
-public class TwitterCharSequenceNormalizer implements CharSequenceNormalizer {
-
- private static final long serialVersionUID = -8155452559337913929L;
-
- private static final Pattern HASH_USER_REGEX =
- Pattern.compile("[#@]\\S+");
-
- private static final Pattern RT_REGEX =
- Pattern.compile("\\b(rt[ :])+", Pattern.CASE_INSENSITIVE);
-
- private static final Pattern FACE_REGEX =
- Pattern.compile("[:;x]-?[()dop]", Pattern.CASE_INSENSITIVE);
-
- private static final Pattern LAUGH_REGEX =
- Pattern.compile("([hj])+([aieou])+(\\1+\\2+)+",
Pattern.CASE_INSENSITIVE);
-
- private static final TwitterCharSequenceNormalizer INSTANCE = new
TwitterCharSequenceNormalizer();
-
- public static TwitterCharSequenceNormalizer getInstance() {
- return INSTANCE;
- }
-
- @Override
- public CharSequence normalize (CharSequence text) {
- String modified = HASH_USER_REGEX.matcher(text).replaceAll(" ");
- modified = RT_REGEX.matcher(modified).replaceAll(" ");
- modified = FACE_REGEX.matcher(modified).replaceAll(" ");
- modified = LAUGH_REGEX.matcher(modified).replaceAll("$1$2$1$2");
- return modified;
- }
-}
diff --git
a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/UrlCharSequenceNormalizer.java
b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/UrlCharSequenceNormalizer.java
index 67d6cb167..f843f1109 100644
---
a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/UrlCharSequenceNormalizer.java
+++
b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/UrlCharSequenceNormalizer.java
@@ -16,29 +16,201 @@
*/
package opennlp.tools.util.normalizer;
-import java.util.regex.Pattern;
-
/**
- * A {@link UrlCharSequenceNormalizer} implementation that normalizes text
+ * A {@link CharSequenceNormalizer} implementation that normalizes text
* in terms of URls and email addresses. Every encounter will be replaced by a
whitespace.
+ *
+ * <p>Normalization runs in two passes:</p>
+ * <ol>
+ * <li>URLs: a lowercase {@code http://} or {@code https://} scheme followed
by at least one
+ * character out of the body set {@code [-_.?&~;+=/#0-9A-Za-z]} becomes
one space; the
+ * match ends at the first character outside that set, so a colon
(port), a percent
+ * escape, an at sign (userinfo), or a non-ASCII label cuts it
short.</li>
+ * <li>Email addresses: a maximal run of the local-part set {@code
[-+_.0-9A-Za-z]} whose
+ * left neighbor is outside that set, an {@code @}, and a domain run out
of
+ * {@code [-.0-9A-Za-z]} that must not start with a dot and must span at
least two
+ * chars, become one space.</li>
+ * </ol>
*/
public class UrlCharSequenceNormalizer implements CharSequenceNormalizer {
private static final long serialVersionUID = 2023145028634552389L;
- private static final Pattern URL_REGEX =
- Pattern.compile("https?://[-_.?&~;+=/#0-9A-Za-z]+");
- private static final Pattern MAIL_REGEX =
-
Pattern.compile("(?<![-+_.0-9A-Za-z])[-+_.0-9A-Za-z]+@[-0-9A-Za-z]+[-.0-9A-Za-z]+");
+
+ private static final CodePointSet ASCII_ALNUM = CodePointSet.ofRange('0',
'9')
+ .union(CodePointSet.ofRange('A', 'Z'))
+ .union(CodePointSet.ofRange('a', 'z'));
+
+ /** The URL body set: {@code [-_.?&~;+=/#0-9A-Za-z]}. */
+ private static final CodePointSet URL_BODY =
+ ASCII_ALNUM.union(CodePointSet.of('-', '_', '.', '?', '&', '~', ';',
'+', '=', '/', '#'));
+
+ /** The mail local-part set, also the left-neighbor exclusion set: {@code
[-+_.0-9A-Za-z]}. */
+ private static final CodePointSet MAIL_LOCAL =
+ ASCII_ALNUM.union(CodePointSet.of('-', '+', '_', '.'));
+
+ /** The set a domain may start with: {@code [-0-9A-Za-z]}. */
+ private static final CodePointSet MAIL_DOMAIN_START =
ASCII_ALNUM.union(CodePointSet.of('-'));
+
+ /** The set a domain continues with: {@code [-.0-9A-Za-z]}. */
+ private static final CodePointSet MAIL_DOMAIN =
MAIL_DOMAIN_START.union(CodePointSet.of('.'));
private static final UrlCharSequenceNormalizer INSTANCE = new
UrlCharSequenceNormalizer();
+ /** {@return the shared, stateless instance} */
public static UrlCharSequenceNormalizer getInstance() {
return INSTANCE;
}
+ /**
+ * {@inheritDoc}
+ */
@Override
- public CharSequence normalize (CharSequence text) {
- String modified = URL_REGEX.matcher(text).replaceAll(" ");
- return MAIL_REGEX.matcher(modified).replaceAll(" ");
+ public CharSequence normalize(CharSequence text) {
+ if (text == null) {
+ throw new IllegalArgumentException("The text must not be null.");
+ }
+ return removeMailAddresses(removeUrls(text));
+ }
+
+ /**
+ * Replaces each URL with one space.
+ *
+ * @param text The text to scan; never null.
+ * @return The input itself when nothing matched, otherwise the normalized
copy.
+ */
+ private CharSequence removeUrls(CharSequence text) {
+ final int length = text.length();
+ StringBuilder out = null;
+ int i = 0;
+ while (i < length) {
+ final int end = matchUrlEnd(text, i);
+ if (end > i) {
+ if (out == null) {
+ out = new StringBuilder(length).append(text, 0, i);
+ }
+ out.append(' ');
+ i = end;
+ } else {
+ if (out != null) {
+ out.append(text.charAt(i));
+ }
+ i++;
+ }
+ }
+ return out == null ? text : out.toString();
+ }
+
+ /**
+ * {@return the exclusive end of a URL match starting at {@code start}, or
{@code -1} if
+ * there is none}
+ *
+ * @param text The text to look into; never null.
+ * @param start The index the match must start at.
+ */
+ private int matchUrlEnd(CharSequence text, int start) {
+ final int length = text.length();
+ if (!regionEquals(text, start, "http")) {
+ return -1;
+ }
+ int bodyStart = start + 4;
+ if (bodyStart < length && text.charAt(bodyStart) == 's'
+ && regionEquals(text, bodyStart + 1, "://")) {
+ bodyStart += 4;
+ } else if (regionEquals(text, bodyStart, "://")) {
+ bodyStart += 3;
+ } else {
+ return -1;
+ }
+ if (bodyStart >= length || !URL_BODY.contains(text.charAt(bodyStart))) {
+ return -1;
+ }
+ int end = bodyStart + 1;
+ while (end < length && URL_BODY.contains(text.charAt(end))) {
+ end++;
+ }
+ return end;
+ }
+
+ /**
+ * {@return whether the chars at {@code at} equal {@code literal} exactly}
+ *
+ * @param text The text to look into; never null.
+ * @param at The index the comparison starts at.
+ * @param literal The chars to compare against; never null.
+ */
+ private boolean regionEquals(CharSequence text, int at, String literal) {
+ if (at + literal.length() > text.length()) {
+ return false;
+ }
+ for (int k = 0; k < literal.length(); k++) {
+ if (text.charAt(at + k) != literal.charAt(k)) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ /**
+ * Replaces each email address with one space.
+ *
+ * @param text The text to scan; never null.
+ * @return The input itself when nothing matched, otherwise the normalized
copy.
+ */
+ private CharSequence removeMailAddresses(CharSequence text) {
+ final int length = text.length();
+ StringBuilder out = null;
+ int i = 0;
+ while (i < length) {
+ final int end = matchMailEnd(text, i);
+ if (end > i) {
+ if (out == null) {
+ out = new StringBuilder(length).append(text, 0, i);
+ }
+ out.append(' ');
+ i = end;
+ } else {
+ if (out != null) {
+ out.append(text.charAt(i));
+ }
+ i++;
+ }
+ }
+ return out == null ? text : out.toString();
+ }
+
+ /**
+ * {@return the exclusive end of an email match starting at {@code start},
or {@code -1} if
+ * there is none}
+ *
+ * @param text The text to look into; never null.
+ * @param start The index the match must start at.
+ */
+ private int matchMailEnd(CharSequence text, int start) {
+ if (start > 0 && MAIL_LOCAL.contains(text.charAt(start - 1))) {
+ return -1; // a match never starts inside a local-part run
+ }
+ if (!MAIL_LOCAL.contains(text.charAt(start))) {
+ return -1;
+ }
+ final int length = text.length();
+ int at = start + 1;
+ while (at < length && MAIL_LOCAL.contains(text.charAt(at))) {
+ at++;
+ }
+ if (at >= length || text.charAt(at) != '@') {
+ return -1;
+ }
+ final int domainStart = at + 1;
+ if (domainStart >= length ||
!MAIL_DOMAIN_START.contains(text.charAt(domainStart))) {
+ return -1;
+ }
+ int end = domainStart + 1;
+ while (end < length && MAIL_DOMAIN.contains(text.charAt(end))) {
+ end++;
+ }
+ if (end - domainStart < 2) {
+ return -1;
+ }
+ return end;
}
}
diff --git
a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/CharSequenceNormalizerContractTest.java
b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/CharSequenceNormalizerContractTest.java
new file mode 100644
index 000000000..40e537d49
--- /dev/null
+++
b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/CharSequenceNormalizerContractTest.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.tools.util.normalizer;
+
+import java.util.stream.Stream;
+
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+/**
+ * The {@link CharSequenceNormalizer} contract held against every
implementation in this
+ * package: null input throws {@link IllegalArgumentException}, with no
exceptions and no
+ * legacy carve-outs.
+ */
+class CharSequenceNormalizerContractTest {
+
+ static Stream<Arguments> allNormalizers() {
+ return Stream.of(
+ Arguments.of("accentFold",
AccentFoldCharSequenceNormalizer.getInstance()),
+ Arguments.of("aggregate",
+ new
AggregateCharSequenceNormalizer(NfcCharSequenceNormalizer.getInstance())),
+ Arguments.of("caseFold", CaseFoldCharSequenceNormalizer.getInstance()),
+ Arguments.of("confusableSkeleton",
ConfusableSkeletonCharSequenceNormalizer.getInstance()),
+ Arguments.of("emoji", EmojiCharSequenceNormalizer.getInstance()),
+ Arguments.of("nfc", NfcCharSequenceNormalizer.getInstance()),
+ Arguments.of("nfkc", NfkcCharSequenceNormalizer.getInstance()),
+ Arguments.of("number", NumberCharSequenceNormalizer.getInstance()),
+ Arguments.of("shrink", ShrinkCharSequenceNormalizer.getInstance()),
+ Arguments.of("socialMedia",
SocialMediaCharSequenceNormalizer.getInstance()),
+ Arguments.of("url", UrlCharSequenceNormalizer.getInstance()),
+ Arguments.of("bullet", BulletCharSequenceNormalizer.getInstance()),
+ Arguments.of("dash", DashCharSequenceNormalizer.getInstance()),
+ Arguments.of("digit", DigitCharSequenceNormalizer.getInstance()),
+ Arguments.of("ellipsis", EllipsisCharSequenceNormalizer.getInstance()),
+ Arguments.of("germanUmlaut",
GermanUmlautCharSequenceNormalizer.getInstance()),
+ Arguments.of("invisible",
InvisibleCharSequenceNormalizer.getInstance()),
+ Arguments.of("lineBreakPreservingWhitespace",
+ LineBreakPreservingWhitespaceCharSequenceNormalizer.getInstance()),
+ Arguments.of("quote", QuoteCharSequenceNormalizer.getInstance()),
+ Arguments.of("whitespace",
WhitespaceCharSequenceNormalizer.getInstance()));
+ }
+
+ @ParameterizedTest(name = "{0}")
+ @MethodSource("allNormalizers")
+ void testNullTextThrowsIllegalArgumentException(String name,
+ CharSequenceNormalizer
normalizer) {
+ assertThrows(IllegalArgumentException.class, () ->
normalizer.normalize(null),
+ name + " must reject null per the CharSequenceNormalizer contract");
+ }
+
+ @ParameterizedTest(name = "{0}")
+ @MethodSource("allNormalizers")
+ void testEmptyTextIsAccepted(String name, CharSequenceNormalizer normalizer)
{
+ assertEquals("", normalizer.normalize("").toString(),
+ name + " must pass empty text through");
+ }
+
+ @Test
+ void testNumberJavadocExample() {
+ assertEquals("a b ",
+
NumberCharSequenceNormalizer.getInstance().normalize("a1234b56").toString());
+ }
+
+ @Test
+ void testShrinkJavadocExample() {
+ assertEquals("cool",
+
ShrinkCharSequenceNormalizer.getInstance().normalize("coooool").toString());
+ }
+}
diff --git
a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/CharacterizationInputs.java
b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/CharacterizationInputs.java
new file mode 100644
index 000000000..3511a6f2b
--- /dev/null
+++
b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/CharacterizationInputs.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.tools.util.normalizer;
+
+import java.util.Random;
+
+/**
+ * Input construction shared by the characterization tests of the de-regexed
normalizers, so the
+ * generation and failure-reporting logic cannot drift apart between the four
suites.
+ */
+final class CharacterizationInputs {
+
+ private CharacterizationInputs() {
+ }
+
+ static String randomInput(Random random, String[] pool) {
+ final int pieces = random.nextInt(24);
+ final StringBuilder b = new StringBuilder();
+ for (int i = 0; i < pieces; i++) {
+ b.append(pool[random.nextInt(pool.length)]);
+ }
+ return b.toString();
+ }
+
+ static String escape(String s) {
+ final StringBuilder b = new StringBuilder();
+ for (int i = 0; i < s.length(); i++) {
+ final char c = s.charAt(i);
+ if (c >= 0x20 && c <= 0x7E) {
+ b.append(c);
+ } else {
+ b.append(String.format("\\u%04X", (int) c));
+ }
+ }
+ return b.toString();
+ }
+}
diff --git
a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/NumberCharSequenceNormalizerCharacterizationTest.java
b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/NumberCharSequenceNormalizerCharacterizationTest.java
new file mode 100644
index 000000000..16a7ca385
--- /dev/null
+++
b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/NumberCharSequenceNormalizerCharacterizationTest.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.tools.util.normalizer;
+
+import java.util.Random;
+import java.util.regex.Pattern;
+
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertSame;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+/**
+ * Characterization tests for {@link NumberCharSequenceNormalizer}.
+ *
+ * <p>The fixed expectations below were probed against the regex implementation
+ * ({@code "\\d+"} replaced by a single space) and pin its output byte for
byte: every maximal run
+ * of ASCII digits collapses to one space, and nothing else changes. In
particular, non-ASCII
+ * digits are left alone, because the regex {@code \d} class (without
+ * {@code UNICODE_CHARACTER_CLASS}) only ever matched {@code [0-9]}.</p>
+ */
+public class NumberCharSequenceNormalizerCharacterizationTest {
+
+ private static final NumberCharSequenceNormalizer NORMALIZER =
+ NumberCharSequenceNormalizer.getInstance();
+
+ private static void check(String input, String expected) {
+ assertEquals(expected, NORMALIZER.normalize(input).toString());
+ }
+
+ @Test
+ void asciiDigitRunsCollapseToASingleSpace() {
+ check("", "");
+ check("abc", "abc");
+ check("123", " ");
+ check("absc 123,0123 abcd", "absc , abcd");
+ check("a1b22c333d", "a b c d");
+ check(" 12 34 ", " ");
+ check("0", " ");
+ check("12.34", " . ");
+ check("-5", "- ");
+ check("12ab34", " ab ");
+ check("x0123456789y", "x y");
+ check("1", " ");
+ check("a1", "a ");
+ check("1a", " a");
+ check(" 123 ", " ");
+ check("phone: +1-800-555-0199", "phone: + - - - ");
+ }
+
+ @Test
+ void nonAsciiDigitsAreNotTouched() {
+ // Arabic-Indic digits, a Bengali digit, a circled digit, and the
mathematical bold digit one
+ // (a supplementary code point) are all outside the ASCII \d class.
+ check("\u0661\u0662\u0663", "\u0661\u0662\u0663");
+ check("\u09E9", "\u09E9");
+ check("\u2460", "\u2460");
+ check("\uD835\uDFCF", "\uD835\uDFCF");
+ check("\uD83D\uDE001\uD83D\uDE00", "\uD83D\uDE00 \uD83D\uDE00");
+ check("1\uD835\uDFCF2", " \uD835\uDFCF ");
+ }
+
+ @Test
+ void digitFreeInputIsReturnedUncopied() {
+ final String text = "no digits in here at all";
+ assertSame(text,
NumberCharSequenceNormalizer.getInstance().normalize(text));
+ }
+
+ @Test
+ void nullTextIsRejected() {
+ // Null input is rejected with IllegalArgumentException.
+ assertThrows(IllegalArgumentException.class, () ->
NORMALIZER.normalize(null));
+ }
+
+ @Test
+ void matchesTheFormerRegexOnRandomizedInputs() {
+ final Pattern formerRegex = Pattern.compile("\\d+");
+ final String[] pool = {"a", "Z", " ", "0", "1", "23", "007", ".", ",",
"-", "+",
+ "\u0661", "\u2460", "\uD835\uDFCF", "\uD83D\uDE00", "\uD83D",
"\uDE00"};
+ final Random random = new Random(42);
+ for (int i = 0; i < 5000; i++) {
+ final String input = CharacterizationInputs.randomInput(random, pool);
+ assertEquals(formerRegex.matcher(input).replaceAll(" "),
+ NORMALIZER.normalize(input).toString(), () -> "Input: " +
CharacterizationInputs.escape(input));
+ }
+ }
+}
diff --git
a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/ShrinkCharSequenceNormalizerCharacterizationTest.java
b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/ShrinkCharSequenceNormalizerCharacterizationTest.java
new file mode 100644
index 000000000..81fb87d59
--- /dev/null
+++
b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/ShrinkCharSequenceNormalizerCharacterizationTest.java
@@ -0,0 +1,166 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.tools.util.normalizer;
+
+import java.util.Random;
+import java.util.regex.Pattern;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+/**
+ * Characterization tests for {@link ShrinkCharSequenceNormalizer}.
+ *
+ * <p>The fixed expectations below were probed against the regex implementation
+ * ({@code "\\s{2,}"} replaced by one space, then {@code "(.)\\1{2,}"} with
+ * {@code CASE_INSENSITIVE} replaced by {@code "$1$1"}, then {@link
String#trim()}) and pin its
+ * output byte for byte. The behavior has several sharp edges worth keeping
visible:</p>
+ * <ul>
+ * <li>{@code \s} is the ASCII class, so only runs of the six ASCII
whitespace characters
+ * collapse; a single tab survives, and NBSP or NEL never collapse as
whitespace.</li>
+ * <li>The repeated-character pass is ASCII case-insensitive ({@code "aAAb"}
shrinks to
+ * {@code "aab"}, keeping the first character's case twice), but it
never starts a run on a
+ * regex line terminator because {@code .} does not match one.</li>
+ * <li>Supplementary-plane repeats shrink as whole code points, and lone
surrogates follow the
+ * regex engine's code-point reading of the input.</li>
+ * </ul>
+ */
+public class ShrinkCharSequenceNormalizerCharacterizationTest {
+
+ private static final ShrinkCharSequenceNormalizer NORMALIZER =
+ ShrinkCharSequenceNormalizer.getInstance();
+
+ private static void check(String input, String expected) {
+ assertEquals(expected, NORMALIZER.normalize(input).toString());
+ }
+
+ @Test
+ void whitespaceRunsOfTwoOrMoreCollapseToOneSpace() {
+ check("", "");
+ check(" ", "");
+ check("a text extra space", "a text extra space");
+ check("a b", "a b");
+ check("a \t b", "a b");
+ check("a\tb", "a\tb");
+ check("a\t\tb", "a b");
+ check("\tab", "ab");
+ check("ab\t", "ab");
+ check("a\n\n\nb", "a b");
+ check("a\nb", "a\nb");
+ check("a\r\n\r\nb", "a b");
+ check(" \t\n ", "");
+ check("x X x X", "x X x X");
+ }
+
+ @Test
+ void repeatedCharacterRunsShrinkToTwoAsciiCaseInsensitively() {
+ check("Helllllloooooo", "Helloo");
+ check("Hello", "Hello");
+ check("HHello", "HHello");
+ check("aaab", "aab");
+ check("aaaa", "aa");
+ check("aAAb", "aab");
+ check("aAaB", "aaB");
+ check("aAa", "aa");
+ check("aA", "aA");
+ check("heyyy", "heyy");
+ check("cooool!!!!", "cool!!");
+ check("....", "..");
+ check("----", "--");
+ check("aa", "aa");
+ check("SSSs", "SS");
+ check("111222333", "112233");
+ check("ababab", "ababab");
+ check("xxxXXX", "xx");
+ check("aaabbb ccc", "aabb cc");
+ check("aaa\n\n\nbbb", "aa bb");
+ check("a bbb c", "a bb c");
+ }
+
+ @Test
+ void edgesAreTrimmedLikeStringTrim() {
+ check(" aaa ", "aa");
+ check("a", "a");
+ check("a ", "a");
+ // String.trim() drops every char up to U+0020, not only whitespace.
+ check("\u0001a\u0001", "a");
+ }
+
+ @Test
+ void regexLineTerminatorsNeverStartARepeatRun() {
+ // NEL, LINE SEPARATOR, and PARAGRAPH SEPARATOR are not in the ASCII \s
class, and the
+ // regex "." refused to match them, so their runs survived both passes
(and trim keeps
+ // them, as they are above U+0020).
+ check("\u0085\u0085\u0085", "\u0085\u0085\u0085");
+ check("\u2028\u2028\u2028", "\u2028\u2028\u2028");
+ check("\u2029\u2029\u2029", "\u2029\u2029\u2029");
+ // NBSP is matched by "." and shrinks like any repeated character.
+ check("\u00A0\u00A0\u00A0", "\u00A0\u00A0");
+ }
+
+ @Test
+ void nonAsciiRepeatsCompareByExactCodePoint() {
+ // ASCII-only case folding: e-acute repeats shrink, but E-acute does not
fold to e-acute.
+ check("\u00E9\u00E9\u00E9\u00E9", "\u00E9\u00E9");
+ check("\u00C9\u00C9\u00E9\u00E9", "\u00C9\u00C9\u00E9\u00E9");
+ }
+
+ @Test
+ void supplementaryAndLoneSurrogateRunsFollowTheRegexReading() {
+ check("\uD83D\uDE00\uD83D\uDE00\uD83D\uDE00", "\uD83D\uDE00\uD83D\uDE00");
+ check("\uD83D\uDE00\uD83D\uDE00\uD83D\uDE00\uD83D\uDE00",
"\uD83D\uDE00\uD83D\uDE00");
+ // Three lone high surrogates before a low one do not shrink: at the third
one the engine
+ // reads a full surrogate pair, which no longer equals the one-char group.
+ check("\uD83D\uD83D\uD83D\uDE00", "\uD83D\uD83D\uD83D\uDE00");
+ // A pair followed by two lone low surrogates shrinks the
three-low-surrogate run found
+ // when scanning resumes inside the pair.
+ check("\uD83D\uDE00\uDE00\uDE00", "\uD83D\uDE00\uDE00");
+ }
+
+ @Test
+ void nullTextIsRejected() {
+ // Null input is rejected with IllegalArgumentException.
+ assertThrows(IllegalArgumentException.class, () ->
NORMALIZER.normalize(null));
+ }
+
+ @Test
+ void matchesTheFormerRegexOnRandomizedInputs() {
+ final Pattern spaceRegex = Pattern.compile("\\s{2,}",
Pattern.CASE_INSENSITIVE);
+ final Pattern repeatedCharRegex = Pattern.compile("(.)\\1{2,}",
Pattern.CASE_INSENSITIVE);
+ final String[] pool = {"a", "aa", "aaa", "A", "b", "Z", "!", "!!", ".",
"-", " ", " ",
+ "\t", "\n", "\r", "\u0001", "\u0085", "\u2028", "\u00A0", "\u00E9",
"\u00C9",
+ "\uD83D\uDE00", "\uD83D", "\uDE00"};
+ final Random random = new Random(42);
+ for (int i = 0; i < 5000; i++) {
+ final String input = CharacterizationInputs.randomInput(random, pool);
+ final String expected = repeatedCharRegex
+ .matcher(spaceRegex.matcher(input).replaceAll(" "))
+ .replaceAll("$1$1").trim();
+ assertEquals(expected, NORMALIZER.normalize(input).toString(),
+ () -> "Input: " + CharacterizationInputs.escape(input));
+ }
+ }
+ @Test
+ void noMatchInputIsReturnedUncopied() {
+ // Single spaces, no repeat runs, nothing to trim: no pass may allocate.
+ final String plain = "single spaced words only";
+ Assertions.assertSame(plain, NORMALIZER.normalize(plain));
+ }
+}
diff --git
a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/SocialMediaCharSequenceNormalizerCharacterizationTest.java
b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/SocialMediaCharSequenceNormalizerCharacterizationTest.java
new file mode 100644
index 000000000..b53be6d26
--- /dev/null
+++
b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/SocialMediaCharSequenceNormalizerCharacterizationTest.java
@@ -0,0 +1,198 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.tools.util.normalizer;
+
+import java.util.Random;
+import java.util.regex.Pattern;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+/**
+ * Characterization tests for {@link SocialMediaCharSequenceNormalizer}.
+ *
+ * <p>The fixed expectations below were probed against the regex
implementation (four passes:
+ * {@code "[#@]\\S+"} to a space, {@code "\\b(rt[ :])+"} case-insensitive to a
space,
+ * {@code "[:;x]-?[()dop]"} case-insensitive to a space, and the laugh pattern
+ * {@code "([hj])+([aieou])+(\\1+\\2+)+"} case-insensitive to {@code
"$1$2$1$2"}) and pin the
+ * output byte for byte. Notable sharp edges kept visible here:</p>
+ * <ul>
+ * <li>{@code \S} is the complement of the six ASCII whitespace characters,
so a hashtag or
+ * handle swallows NBSP, emoji, and any other non-ASCII-space
characters.</li>
+ * <li>The word boundary before {@code rt} follows the JDK engine: ASCII
word characters block
+ * it, and so does a non-spacing mark that has a base character,
scanning backwards char by
+ * char (which makes a supplementary-plane mark behave as if it had no
base).</li>
+ * <li>The emoticon pass happily fires inside words ({@code "expo"} contains
{@code "xp"}).</li>
+ * <li>The laugh pass keeps the case of the last consonant and last vowel of
the two leading
+ * runs ({@code "hAHa"} becomes {@code "hAhA"}, {@code "hjaja"} becomes
{@code "jaja"}).</li>
+ * </ul>
+ */
+// The word-boundary characterization pins the JDK 21+ regex engine's
non-ASCII behavior; on
+// older engines "\b" treated non-ASCII word characters differently, so these
expectations are
+// tied to the project's Java baseline.
+public class SocialMediaCharSequenceNormalizerCharacterizationTest {
+
+ private static final SocialMediaCharSequenceNormalizer NORMALIZER =
+ SocialMediaCharSequenceNormalizer.getInstance();
+
+ private static void check(String input, String expected) {
+ assertEquals(expected, NORMALIZER.normalize(input).toString());
+ }
+
+ @Test
+ void hashtagsAndHandlesBecomeASingleSpace() {
+ check("", "");
+ check("asdf #hasdk23 2nnfdf", "asdf 2nnfdf");
+ check("asdf @hasdk23 2nnfdf", "asdf 2nnfdf");
+ check("#", "#");
+ check("##", " ");
+ check("# #", "# #");
+ check("#a#b", " ");
+ check("abc#def", "abc ");
+ check("@user hi", " hi");
+ check("@ home", "@ home");
+ check("#\uD83D\uDE00", " ");
+ check("#\u00A0x", " ");
+ check("email me @home now", "email me now");
+ check("\uD83D\uDE00 #yes \uD83D\uDE00", "\uD83D\uDE00 \uD83D\uDE00");
+ }
+
+ @Test
+ void retweetMarkersCollapseBehindAJdkStyleWordBoundary() {
+ check("RT RT RT 2nnfdf", " 2nnfdf");
+ check("rt this", " this");
+ check("rt rt rt", " rt");
+ check("RT: hello", " hello");
+ check("rt:rt:go", " go");
+ check("RT @user: hello", " hello");
+ check("art here", "art here");
+ check("smart tv", "smart tv");
+ check("cart rt cart", "cart cart");
+ check("hrt x", "hrt x");
+ check("1rt x", "1rt x");
+ check("_rt x", "_rt x");
+ check("-rt x", "- x");
+ }
+
+ @Test
+ void wordBoundaryTreatsNonAsciiLettersAsNonWordButHonorsCombiningMarks() {
+ // e-acute is not an ASCII word char, so "rt" after it sits on a boundary
and matches.
+ check("\u00E9rt x", "\u00E9 x");
+ // A combining mark with a base character blocks the boundary, like the
JDK engine.
+ check("e\u0301rt x", "e\u0301rt x");
+ // A combining mark with no base character does not block it.
+ check("\u0301rt x", "\u0301 x");
+ // A supplementary-plane mark: the engine's backwards scan reads a lone
low surrogate and
+ // stops, so the mark does not block the boundary despite its base
character.
+ check("a\uD800\uDDFDrt x", "a\uD800\uDDFD x");
+ }
+
+ @Test
+ void emoticonsBecomeASingleSpaceEvenInsideWords() {
+ check("hello :-) hello", "hello hello");
+ check("hello ;) hello", "hello hello");
+ check(":) hello", " hello");
+ check("hello :P", "hello ");
+ check("x-d ok", " ok");
+ check(":-( sad", " sad");
+ check("::-)", ": ");
+ check("expo", "e o");
+ check("xoxo", " ");
+ check("taxi", "taxi");
+ check(":-x", ":-x");
+ check("X-P", " ");
+ check(":o", " ");
+ check(":d", " ");
+ check("x-o", " ");
+ check("xp", " ");
+ check("max power", "max power");
+ }
+
+ @Test
+ void laughterShrinksToTheLastConsonantVowelPairTwice() {
+ check("ahahahah", "ahahah");
+ check("hahha", "haha");
+ check("hahaa", "haha");
+ check("ahahahahhahahhahahaaaa", "ahaha");
+ check("jajjajajaja", "jaja");
+ check("hahaha", "haha");
+ check("haha", "haha");
+ check("HaHaHa", "HaHa");
+ check("hAHa", "hAhA");
+ check("HAHAHA", "HAHA");
+ check("hhaahhaa", "haha");
+ check("hjaja", "jaja");
+ check("ha", "ha");
+ check("haaaa", "haaaa");
+ check("hahe", "hahe");
+ check("hajaha", "hajaha");
+ check("jejeje", "jeje");
+ check("Bahahaha", "Bahaha");
+ check("hihihi", "hihi");
+ check("hohoho", "hoho");
+ check("huhuhu", "huhu");
+ check("hyhy", "hyhy");
+ check("jujuju", "juju");
+ check("hahaha!", "haha!");
+ check("hjhj", "hjhj");
+ }
+
+ @Test
+ void thePassesComposeInOrder() {
+ check("RT @user: check #cool :-) hahaha", " check haha");
+ check("x #tag rt go", "x go");
+ }
+
+ @Test
+ void nullTextIsRejected() {
+ // Null input is rejected with IllegalArgumentException.
+ assertThrows(IllegalArgumentException.class, () ->
NORMALIZER.normalize(null));
+ }
+
+ @Test
+ void matchesTheFormerRegexesOnRandomizedInputs() {
+ final Pattern hashUserRegex = Pattern.compile("[#@]\\S+");
+ final Pattern rtRegex = Pattern.compile("\\b(rt[ :])+",
Pattern.CASE_INSENSITIVE);
+ final Pattern faceRegex = Pattern.compile("[:;x]-?[()dop]",
Pattern.CASE_INSENSITIVE);
+ final Pattern laughRegex =
+ Pattern.compile("([hj])+([aieou])+(\\1+\\2+)+",
Pattern.CASE_INSENSITIVE);
+ final String[] pool = {"a", "A", "e", "h", "H", "j", "J", "o", "u", "y",
"x", "X", "d", "p",
+ "P", ")", "(", "-", ":", ";", " ", " ", "\t", "\n", "#", "@", "_",
"1", "rt", "RT",
+ "rt:", "Rt ", "t", "r", "ha", "Ha", "ah", "hh", "aa", "jj", "\u00E9",
"\u0301", "\uD800\uDDFD",
+ "\uD83D\uDE00", "\uD83D", "\uDE00", "\u00A0"};
+ final Random random = new Random(42);
+ for (int i = 0; i < 5000; i++) {
+ final String input = CharacterizationInputs.randomInput(random, pool);
+ String expected = hashUserRegex.matcher(input).replaceAll(" ");
+ expected = rtRegex.matcher(expected).replaceAll(" ");
+ expected = faceRegex.matcher(expected).replaceAll(" ");
+ expected = laughRegex.matcher(expected).replaceAll("$1$2$1$2");
+ assertEquals(expected, NORMALIZER.normalize(input).toString(),
+ () -> "Input: " + CharacterizationInputs.escape(input));
+ }
+ }
+ @Test
+ void noMatchInputIsReturnedUncopied() {
+ // With nothing to fold, every pass declines to allocate and the input
instance comes back.
+ final String plain = "the bird flew over green fields";
+ Assertions.assertSame(plain, NORMALIZER.normalize(plain));
+ }
+
+}
diff --git
a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/UrlCharSequenceNormalizerCharacterizationTest.java
b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/UrlCharSequenceNormalizerCharacterizationTest.java
new file mode 100644
index 000000000..2898452f6
--- /dev/null
+++
b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/UrlCharSequenceNormalizerCharacterizationTest.java
@@ -0,0 +1,204 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.tools.util.normalizer;
+
+import java.util.Random;
+import java.util.regex.Pattern;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+/**
+ * Characterization tests for {@link UrlCharSequenceNormalizer}.
+ *
+ * <p>The fixed expectations below were probed against the regex implementation
+ * ({@code "https?://[-_.?&~;+=/#0-9A-Za-z]+"} to a space, then
+ * {@code "(?<![-+_.0-9A-Za-z])[-+_.0-9A-Za-z]+@[-0-9A-Za-z]+[-.0-9A-Za-z]+"}
to a space) and pin
+ * the accept/reject boundary byte for byte. Notable sharp edges kept
visible:</p>
+ * <ul>
+ * <li>the scheme is lowercase-only, and a URL body needs at least one
character out of the
+ * body class, so {@code "http:// x"} and {@code "http://"} pass through
untouched;</li>
+ * <li>the body class has no colon, percent sign, at sign, or non-ASCII, so
ports, escapes,
+ * userinfo, and IDN labels cut the match short;</li>
+ * <li>a mail match needs a local part run bounded on the left by the
lookbehind, and a domain
+ * of at least two characters starting with no dot: {@code "a@b c"} does
not match, while
+ * {@code "a@b."} swallows the trailing dot;</li>
+ * <li>after a failed candidate the scan resumes one char further, so {@code
"a@[email protected]"}
+ * still finds the inner address.</li>
+ * </ul>
+ */
+public class UrlCharSequenceNormalizerCharacterizationTest {
+
+ private static final UrlCharSequenceNormalizer NORMALIZER =
+ UrlCharSequenceNormalizer.getInstance();
+
+ private static void check(String input, String expected) {
+ assertEquals(expected, NORMALIZER.normalize(input).toString());
+ }
+
+ @Test
+ void urlsBecomeASingleSpace() {
+ check("", "");
+ check("asdf http://asdf.com/dfa/cxs 2nnfdf", "asdf 2nnfdf");
+ check("asdf http://asdf.com/dfa/cxs 2nnfdf http://asdf.com/dfa/cxs";, "asdf
2nnfdf ");
+ check("http://example.com";, " ");
+ check("visit http://example.com now", "visit now");
+ check("https://a.b/c?d=e&f=g#h+i~j;k=l_m-n";, " ");
+ check("https://x";, " ");
+ check("http://EXAMPLE.com";, " ");
+ check("xhttp://y";, "x ");
+ check("ahttp://x";, "a ");
+ }
+
+ @Test
+ void urlRecognitionRejectsAtTheExactRegexBoundary() {
+ check("http:// x", "http:// x");
+ check("http://";, "http://";);
+ check("HTTP://X.COM", "HTTP://X.COM");
+ check("httpss://x", "httpss://x");
+ check("ftp://x";, "ftp://x";);
+ check("www.example.com", "www.example.com");
+ // The body class has no colon, percent sign, or at sign, and no non-ASCII.
+ check("http://x:8080/y";, " :8080/y");
+ check("http://a%20b";, " %20b");
+ check("http://http://x";, " ://x");
+ check("http://\u043F\u0440\u0438\u043C\u0435\u0440";,
"http://\u043F\u0440\u0438\u043C\u0435\u0440";);
+ check("http://x.\u043F\u0440\u0438\u043C\u0435\u0440";, "
\u043F\u0440\u0438\u043C\u0435\u0440");
+ check("http://[email protected]";, " @example.com");
+ }
+
+ @Test
+ void mailAddressesBecomeASingleSpace() {
+ check("asdf [email protected] 2nnfdf", "asdf 2nnfdf");
+ check("asdf [email protected] 2nnfdf [email protected]", "asdf
2nnfdf ");
+ check("asdf [email protected] 2nnfdf", "asdf 2nnfdf");
+ check("asdf [email protected]_br 2nnfdf", "asdf _br 2nnfdf");
+ check("[email protected]", " ");
+ check("[email protected] end", " end");
+ check("[email protected]", " ");
+ check("[email protected]", " ");
+ check("a@-b", " ");
+ check("[email protected]", " ");
+ check("x@y-.", " ");
+ check("mailto:[email protected]";, "mailto: ");
+ check("[email protected],[email protected]", " , ");
+ check("a [email protected] f", "a f");
+ check("see http://x.y, then [email protected]!", "see , then !");
+ }
+
+ @Test
+ void mailRecognitionRejectsAtTheExactRegexBoundary() {
+ // A one-char domain with nothing to lend to the second domain class fails.
+ check("a@b c", "a@b c");
+ check("a@bc d", " d");
+ // The trailing dot is part of the second domain class and gets swallowed.
+ check("a@b.", " ");
+ check("[email protected].", " ");
+ check("Contact [email protected].", "Contact ");
+ // The domain must not start with a dot.
+ check("[email protected]", "[email protected]");
+ check("[email protected]", "[email protected]");
+ // The local part must be a maximal run: no match can start inside one.
+ check("user@@example.com", "user@@example.com");
+ check("@example.com", "@example.com");
+ // After the failed candidate at "a", the scan resumes and finds
"[email protected]".
+ check("a@[email protected]", "a@ ");
+ }
+
+ @Test
+ void lookbehindOnlySeesTheSingleCharBeforeTheLocalPart() {
+ // Non-ASCII (including a low surrogate) is outside the lookbehind class,
so the address
+ // right after it still matches.
+ check("\uD83D\[email protected]", "\uD83D\uDE00 ");
+ check("\[email protected]", "\u00E9 ");
+ }
+
+ @Test
+ void nullTextIsRejected() {
+ // Null input is rejected with IllegalArgumentException.
+ assertThrows(IllegalArgumentException.class, () ->
NORMALIZER.normalize(null));
+ }
+
+ @Test
+ void matchesTheFormerRegexesOnRandomizedInputs() {
+ final Pattern urlRegex =
Pattern.compile("https?://[-_.?&~;+=/#0-9A-Za-z]+");
+ final Pattern mailRegex =
+
Pattern.compile("(?<![-+_.0-9A-Za-z])[-+_.0-9A-Za-z]+@[-0-9A-Za-z]+[-.0-9A-Za-z]+");
+ final String[] pool = {"a", "b", "Z", "1", " ", ".", ",", "-", "+", "_",
"@", ":", "/", "%",
+ "~", ";", "=", "&", "?", "#", "http", "https", "ttp", "://",
"http://";, "s", "x.com",
+ "co", ".com", "a@b", "@b.co", "\u00E9", "\uD83D\uDE00", "\uD83D",
"\uDE00"};
+ final Random random = new Random(42);
+ for (int i = 0; i < 5000; i++) {
+ final String input = CharacterizationInputs.randomInput(random, pool);
+ final String expected = mailRegex
+ .matcher(urlRegex.matcher(input).replaceAll(" "))
+ .replaceAll(" ");
+ assertEquals(expected, NORMALIZER.normalize(input).toString(),
+ () -> "Input: " + CharacterizationInputs.escape(input));
+ }
+ }
+ @Test
+ void noMatchInputIsReturnedUncopied() {
+ final String plain = "no links in this sentence at all";
+ Assertions.assertSame(plain, NORMALIZER.normalize(plain));
+ }
+
+ @Test
+ void weirdUrlsMatchTheFormerRegexExactly() {
+ // Adversarial shapes in the spirit of the web-platform-tests URL suite:
userinfo, ports,
+ // percent escapes, IPv6 brackets, backslashes, IDN and punycode hosts,
stray schemes and
+ // separators. Each input runs differentially through the two former
patterns, so the
+ // accept/reject boundary is pinned by construction rather than by hand.
+ final Pattern urls = Pattern.compile("https?://[-_.?&~;+=/#0-9A-Za-z]+");
+ final Pattern mails =
+
Pattern.compile("(?<![-+_.0-9A-Za-z])[-+_.0-9A-Za-z]+@[-0-9A-Za-z]+[-.0-9A-Za-z]+");
+ final String[] inputs = {
+ "http://user:[email protected]/path";,
+ "https://example.com:8080/x";,
+ "http://example.com/foo%20bar";,
+ "http://[2001:db8::1]/x";,
+ "http://example.com\\path\\file";,
+ "https://xn--nxasmq6b.example/x";,
+ "http://\u00e9xample.com/x";,
+ "HtTp://Example.com",
+ "//example.com/protocol-relative",
+ "http://example.com.";,
+ "http://example.com?";,
+ "http://a..b//c#frag?q=1;2+3~4";,
+ "mailto:[email protected]";,
+ "a@[email protected]",
+ "user@[1.2.3.4]",
+ "user@%41.com",
+ "[email protected]:25",
+ "data:text/html,x",
+ "javascript:alert(1)",
+ "ftp://example.com/file";,
+ "http://example.com/a b http://second.example/c";,
+ "http:///triple-slash";,
+ "http//missing-colon.example",
+ "https://.leading.dot";,
+ };
+ for (final String input : inputs) {
+ final String viaRegexes =
+ mails.matcher(urls.matcher(input).replaceAll(" ")).replaceAll(" ");
+ assertEquals(viaRegexes, NORMALIZER.normalize(input).toString(), input);
+ }
+ }
+}
diff --git
a/opennlp-extensions/opennlp-spellcheck/src/main/java/opennlp/spellcheck/normalizer/SpellCheckingCharSequenceNormalizer.java
b/opennlp-extensions/opennlp-spellcheck/src/main/java/opennlp/spellcheck/normalizer/SpellCheckingCharSequenceNormalizer.java
index 6f6e16c63..42177e8c4 100644
---
a/opennlp-extensions/opennlp-spellcheck/src/main/java/opennlp/spellcheck/normalizer/SpellCheckingCharSequenceNormalizer.java
+++
b/opennlp-extensions/opennlp-spellcheck/src/main/java/opennlp/spellcheck/normalizer/SpellCheckingCharSequenceNormalizer.java
@@ -19,7 +19,6 @@ package opennlp.spellcheck.normalizer;
import java.util.List;
import java.util.Locale;
-import java.util.Objects;
import java.util.regex.Pattern;
import opennlp.spellcheck.SpellChecker;
@@ -48,57 +47,40 @@ import opennlp.tools.util.normalizer.CharSequenceNormalizer;
* input.</li>
* </ul>
*
- * <p>Several guards keep the corrector from "fixing" tokens that should be
left as
- * they are (configurable through the {@link Builder}):</p>
- * <ul>
- * <li>tokens shorter than {@code minTokenLength} are skipped;</li>
- * <li>numeric tokens are skipped ({@code skipNumbers}, on by default);</li>
- * <li>URL- and email-like tokens are skipped ({@code skipUrls}, on by
default);</li>
- * <li>a token whose lower-cased form is already in the dictionary is never
- * changed (the engine returns it at edit distance {@code 0}).</li>
- * </ul>
+ * <p>Several guards, all configurable through the {@link Builder}, keep the
corrector from
+ * "fixing" tokens that should be left as they are: tokens shorter than {@code
minTokenLength}
+ * are skipped, numeric tokens are skipped when {@code skipNumbers} is set (on
by default),
+ * URL- and email-like tokens are skipped when {@code skipUrls} is set (on by
default), and a
+ * token whose lower-cased form is already in the dictionary is never
changed.</p>
*
- * <p><b>Casing.</b> Dictionaries are normally lower-cased, so lookups are
performed on
- * the lower-cased token, and the original casing pattern is re-applied to the
- * correction: an all-upper token yields an all-upper correction, a
leading-capital
- * token yields a leading-capital correction, otherwise the suggestion's own
casing is
- * used. When no correction applies, the original token (including its casing
and any
- * surrounding punctuation) is emitted unchanged.</p>
+ * <p><b>Casing.</b> Lookups are performed on the lower-cased token and the
original casing
+ * pattern is re-applied to the correction: an all-upper token yields an
all-upper correction,
+ * a leading-capital token yields a leading-capital correction, otherwise the
suggestion's own
+ * casing is used. When no correction applies, the original token is emitted
unchanged.</p>
*
* <p>This normalizer composes cleanly inside an
* {@link AggregateCharSequenceNormalizer}; place it after noise-removing
normalizers
* (URL, emoji, shrink) so it sees clean tokens.</p>
*
- * <p><b>Serialization.</b> {@link CharSequenceNormalizer} is {@link
java.io.Serializable},
- * but the backing {@link SpellChecker} usually is not; it is therefore held
in a
- * {@code transient} field and is {@code null} after Java deserialization. A
deserialized
- * instance is inert until a checker is re-attached: obtain a working copy
with the same
- * settings via {@link #withSpellChecker(SpellChecker)} (this matches how the
engine is
- * rebuilt from a model rather than Java-serialized). Calling {@link
#normalize} on an
- * instance with no checker throws {@link IllegalStateException}.</p>
+ * <p><b>Serialization.</b> The settings fields are serialized, but the backing
+ * {@link SpellChecker} is held in a {@code transient} field and is {@code
null} after Java
+ * deserialization; re-attach a checker with {@link
#withSpellChecker(SpellChecker)} to obtain
+ * a working copy with the same settings. Calling {@link #normalize} on an
instance with no
+ * checker throws {@link IllegalStateException}.</p>
*/
public class SpellCheckingCharSequenceNormalizer implements
CharSequenceNormalizer {
- private static final long serialVersionUID = 1L;
+ private static final long serialVersionUID = 488960282194566688L;
/** The default minimum token length below which tokens are left untouched.
*/
public static final int DEFAULT_MIN_TOKEN_LENGTH = 4;
- /** Matches tokens that are entirely digits, optionally with
grouping/decimal marks. */
- private static final Pattern NUMBER_LIKE =
Pattern.compile("[+-]?[\\d.,]*\\d[\\d.,]*%?");
-
/** Matches URL- and email-like tokens that should never be spell-corrected.
*/
private static final Pattern URL_LIKE = Pattern.compile(
"(?:https?://|www\\.)\\S+"
+ "|[-+_.0-9A-Za-z]+@[-0-9A-Za-z]+\\.[-.0-9A-Za-z]+"
+ "|\\S+\\.(?:com|org|net|edu|gov|io)\\b\\S*");
- /** Leading non-letter/digit run kept verbatim around a token (e.g. opening
quotes). */
- private static final Pattern LEADING_NON_WORD =
Pattern.compile("^[^\\p{L}\\p{N}]+");
-
- /** Trailing non-letter/digit run kept verbatim around a token (e.g.
punctuation). */
- private static final Pattern TRAILING_NON_WORD =
Pattern.compile("[^\\p{L}\\p{N}]+$");
-
/** The correction mode. */
public enum Mode {
/** Correct each whitespace-delimited token independently. */
@@ -133,15 +115,17 @@ public class SpellCheckingCharSequenceNormalizer
implements CharSequenceNormaliz
* @param model the loaded model whose engine is used; must not be {@code
null}
*/
public SpellCheckingCharSequenceNormalizer(SymSpellModel model) {
- this(Objects.requireNonNull(model, "model must not be
null").getSymSpell());
+ this(builder(model));
}
private SpellCheckingCharSequenceNormalizer(Builder b) {
- this.spellChecker = Objects.requireNonNull(b.spellChecker, "spellChecker
must not be null");
+ if (b.spellChecker == null) {
+ throw new IllegalArgumentException("spellChecker must not be null");
+ }
+ this.spellChecker = b.spellChecker;
this.mode = b.mode;
this.minTokenLength = b.minTokenLength;
- // The engine throws if a query exceeds its configured maximum, so clamp
to it; an
- // explicitly smaller requested distance is still honored.
+ // Clamp to the engine's configured maximum; a smaller requested distance
is still honored.
this.maxEditDistance = Math.min(b.maxEditDistance,
this.spellChecker.maxEditDistance());
this.skipNumbers = b.skipNumbers;
this.skipUrls = b.skipUrls;
@@ -152,7 +136,10 @@ public class SpellCheckingCharSequenceNormalizer
implements CharSequenceNormaliz
* @return a new {@link Builder} seeded with sensible defaults
*/
public static Builder builder(SpellChecker spellChecker) {
- return new Builder(Objects.requireNonNull(spellChecker, "spellChecker must
not be null"));
+ if (spellChecker == null) {
+ throw new IllegalArgumentException("spellChecker must not be null");
+ }
+ return new Builder(spellChecker);
}
/**
@@ -160,7 +147,10 @@ public class SpellCheckingCharSequenceNormalizer
implements CharSequenceNormaliz
* @return a new {@link Builder} seeded with sensible defaults
*/
public static Builder builder(SymSpellModel model) {
- return new Builder(Objects.requireNonNull(model, "model must not be
null").getSymSpell());
+ if (model == null) {
+ throw new IllegalArgumentException("model must not be null");
+ }
+ return new Builder(model.getSymSpell());
}
/**
@@ -181,13 +171,17 @@ public class SpellCheckingCharSequenceNormalizer
implements CharSequenceNormaliz
.build();
}
+ /** {@inheritDoc} */
@Override
public CharSequence normalize(CharSequence text) {
if (spellChecker == null) {
throw new IllegalStateException("no SpellChecker attached; this instance
was likely "
+ "restored by Java deserialization. Re-attach one via
withSpellChecker(...).");
}
- if (text == null || text.isEmpty()) {
+ if (text == null) {
+ throw new IllegalArgumentException("The text must not be null.");
+ }
+ if (text.isEmpty()) {
return text;
}
final String input = text.toString();
@@ -197,6 +191,14 @@ public class SpellCheckingCharSequenceNormalizer
implements CharSequenceNormaliz
return normalizePerToken(input);
}
+ /**
+ * Corrects the whole input as a phrase with {@link
SpellChecker#lookupCompound}, repairing
+ * wrongly inserted or omitted spaces. Returns the input unchanged when it
is blank or the
+ * engine offers no correction.
+ *
+ * @param input The text to correct; never null.
+ * @return The corrected phrase, or the input itself when nothing changed.
+ */
private CharSequence normalizeCompound(String input) {
if (input.isBlank()) {
return input;
@@ -209,6 +211,13 @@ public class SpellCheckingCharSequenceNormalizer
implements CharSequenceNormaliz
return corrected.isEmpty() ? input : corrected;
}
+ /**
+ * Corrects each whitespace-delimited token independently, copying the
whitespace runs
+ * between tokens verbatim so the shape of the line is kept.
+ *
+ * @param input The text to correct; never null.
+ * @return The text with each token corrected.
+ */
private CharSequence normalizePerToken(String input) {
final StringBuilder out = new StringBuilder(input.length());
int i = 0;
@@ -240,10 +249,11 @@ public class SpellCheckingCharSequenceNormalizer
implements CharSequenceNormaliz
*/
private String correctToken(String token) {
// Peel off leading/trailing punctuation so the core word is what we look
up.
- final String prefix = match(LEADING_NON_WORD, token);
- final String suffix = token.length() > prefix.length()
- ? match(TRAILING_NON_WORD, token.substring(prefix.length())) : "";
- final String core = token.substring(prefix.length(), token.length() -
suffix.length());
+ final int coreStart = leadingNonWordLength(token);
+ final int coreEnd = token.length() - trailingNonWordLength(token,
coreStart);
+ final String prefix = token.substring(0, coreStart);
+ final String suffix = token.substring(coreEnd);
+ final String core = token.substring(coreStart, coreEnd);
if (!isCorrectable(core)) {
return token;
@@ -274,7 +284,7 @@ public class SpellCheckingCharSequenceNormalizer implements
CharSequenceNormaliz
if (skipUrls && URL_LIKE.matcher(core).matches()) {
return false;
}
- if (skipNumbers && NUMBER_LIKE.matcher(core).matches()) {
+ if (skipNumbers && isNumberLike(core)) {
return false;
}
// A token with no letters at all (pure symbols) cannot be a spelling
error.
@@ -306,6 +316,7 @@ public class SpellCheckingCharSequenceNormalizer implements
CharSequenceNormaliz
return corrected;
}
+ /** {@return whether {@code s} has more than one character and every letter
in it is upper-case} */
private static boolean isAllUpper(String s) {
boolean sawLetter = false;
for (int k = 0; k < s.length(); k++) {
@@ -320,9 +331,89 @@ public class SpellCheckingCharSequenceNormalizer
implements CharSequenceNormaliz
return sawLetter && s.length() > 1;
}
- private static String match(Pattern pattern, String s) {
- final var m = pattern.matcher(s);
- return m.find() ? m.group() : "";
+ /**
+ * {@return the length of the leading run of non-letter, non-number code
points of
+ * {@code token}}
+ *
+ * @param token The token to scan; never null.
+ */
+ private int leadingNonWordLength(String token) {
+ int i = 0;
+ while (i < token.length()) {
+ final int codePoint = token.codePointAt(i);
+ if (isLetterOrNumber(codePoint)) {
+ break;
+ }
+ i += Character.charCount(codePoint);
+ }
+ return i;
+ }
+
+ /**
+ * {@return the length of the trailing run of non-letter, non-number code
points of
+ * {@code token}, looking only behind {@code from}}
+ *
+ * @param token The token to scan; never null.
+ * @param from The index the backward scan must not cross.
+ */
+ private int trailingNonWordLength(String token, int from) {
+ int end = token.length();
+ while (end > from) {
+ final int codePoint = token.codePointBefore(end);
+ if (isLetterOrNumber(codePoint)) {
+ break;
+ }
+ end -= Character.charCount(codePoint);
+ }
+ return token.length() - end;
+ }
+
+ /**
+ * {@return whether {@code codePoint} is a letter or a number} Numbers cover
the decimal,
+ * letter, and other number categories, not only the decimal digits.
+ *
+ * @param codePoint The code point to classify.
+ */
+ private boolean isLetterOrNumber(int codePoint) {
+ if (Character.isLetter(codePoint)) {
+ return true;
+ }
+ final int type = Character.getType(codePoint);
+ return type == Character.DECIMAL_DIGIT_NUMBER
+ || type == Character.LETTER_NUMBER
+ || type == Character.OTHER_NUMBER;
+ }
+
+ /**
+ * {@return whether {@code core} looks like a number: an optional sign, then
digits with
+ * optional grouping or decimal marks containing at least one ASCII digit,
then an optional
+ * trailing percent sign} Package private so the differential test can drive
the
+ * classification directly.
+ *
+ * @param core The token to classify; never null.
+ */
+ boolean isNumberLike(String core) {
+ int start = 0;
+ if (start < core.length() && (core.charAt(start) == '+' ||
core.charAt(start) == '-')) {
+ start++;
+ }
+ int end = core.length();
+ if (end > start && core.charAt(end - 1) == '%') {
+ end--;
+ }
+ if (start >= end) {
+ return false;
+ }
+ boolean sawDigit = false;
+ for (int i = start; i < end; i++) {
+ final char c = core.charAt(i);
+ if (c >= '0' && c <= '9') {
+ sawDigit = true;
+ } else if (c != '.' && c != ',') {
+ return false;
+ }
+ }
+ return sawDigit;
}
/** A mutable builder for {@link SpellCheckingCharSequenceNormalizer}. */
@@ -344,7 +435,10 @@ public class SpellCheckingCharSequenceNormalizer
implements CharSequenceNormaliz
* @return this builder
*/
public Builder mode(Mode value) {
- this.mode = Objects.requireNonNull(value, "mode must not be null");
+ if (value == null) {
+ throw new IllegalArgumentException("mode must not be null");
+ }
+ this.mode = value;
return this;
}
diff --git
a/opennlp-extensions/opennlp-spellcheck/src/main/java/opennlp/spellcheck/stream/SpellCorrectingObjectStream.java
b/opennlp-extensions/opennlp-spellcheck/src/main/java/opennlp/spellcheck/stream/SpellCorrectingObjectStream.java
index 3c3191877..9444d2afd 100644
---
a/opennlp-extensions/opennlp-spellcheck/src/main/java/opennlp/spellcheck/stream/SpellCorrectingObjectStream.java
+++
b/opennlp-extensions/opennlp-spellcheck/src/main/java/opennlp/spellcheck/stream/SpellCorrectingObjectStream.java
@@ -18,7 +18,6 @@
package opennlp.spellcheck.stream;
import java.io.IOException;
-import java.util.Objects;
import opennlp.spellcheck.SpellChecker;
import opennlp.spellcheck.dictionary.SymSpellModel;
@@ -58,10 +57,11 @@ public class SpellCorrectingObjectStream extends
FilterObjectStream<String, Stri
*
* @param samples the source line stream; must not be {@code null}
* @param spellChecker the engine used to correct lines; must not be {@code
null}
+ * @throws IllegalArgumentException if {@code samples} or {@code
spellChecker} is {@code null}
*/
public SpellCorrectingObjectStream(ObjectStream<String> samples,
SpellChecker spellChecker) {
this(samples, new SpellCheckingCharSequenceNormalizer(
- Objects.requireNonNull(spellChecker, "spellChecker must not be
null")));
+ requireNonNullArg(spellChecker, "spellChecker")));
}
/**
@@ -70,10 +70,11 @@ public class SpellCorrectingObjectStream extends
FilterObjectStream<String, Stri
*
* @param samples the source line stream; must not be {@code null}
* @param model the loaded model whose engine is used; must not be {@code
null}
+ * @throws IllegalArgumentException if {@code samples} or {@code model} is
{@code null}
*/
public SpellCorrectingObjectStream(ObjectStream<String> samples,
SymSpellModel model) {
this(samples, new SpellCheckingCharSequenceNormalizer(
- Objects.requireNonNull(model, "model must not be null")));
+ requireNonNullArg(model, "model")));
}
/**
@@ -82,13 +83,33 @@ public class SpellCorrectingObjectStream extends
FilterObjectStream<String, Stri
*
* @param samples the source line stream; must not be {@code null}
* @param normalizer the corrector to apply to each line; must not be {@code
null}
+ * @throws IllegalArgumentException if {@code samples} or {@code normalizer}
is {@code null}
*/
public SpellCorrectingObjectStream(ObjectStream<String> samples,
SpellCheckingCharSequenceNormalizer
normalizer) {
- super(samples);
- this.normalizer = Objects.requireNonNull(normalizer, "normalizer must not
be null");
+ super(requireNonNullArg(samples, "samples"));
+ if (normalizer == null) {
+ throw new IllegalArgumentException("normalizer must not be null");
+ }
+ this.normalizer = normalizer;
+ }
+
+ /**
+ * Validates that a constructor argument is not {@code null}.
+ *
+ * @param value the argument to check
+ * @param name the parameter name used in the error message
+ * @return {@code value}, never {@code null}
+ * @throws IllegalArgumentException if {@code value} is {@code null}
+ */
+ private static <T> T requireNonNullArg(T value, String name) {
+ if (value == null) {
+ throw new IllegalArgumentException(name + " must not be null");
+ }
+ return value;
}
+ /** {@inheritDoc} */
@Override
public String read() throws IOException {
final String line = samples.read();
diff --git
a/opennlp-extensions/opennlp-spellcheck/src/main/java/opennlp/spellcheck/stream/SpellCorrectingTokenStream.java
b/opennlp-extensions/opennlp-spellcheck/src/main/java/opennlp/spellcheck/stream/SpellCorrectingTokenStream.java
index b07aa148c..981368c1a 100644
---
a/opennlp-extensions/opennlp-spellcheck/src/main/java/opennlp/spellcheck/stream/SpellCorrectingTokenStream.java
+++
b/opennlp-extensions/opennlp-spellcheck/src/main/java/opennlp/spellcheck/stream/SpellCorrectingTokenStream.java
@@ -18,8 +18,6 @@
package opennlp.spellcheck.stream;
import java.io.IOException;
-import java.util.Objects;
-import java.util.regex.Pattern;
import opennlp.spellcheck.SpellChecker;
import opennlp.spellcheck.dictionary.SymSpellModel;
@@ -33,11 +31,9 @@ import opennlp.tools.util.ObjectStream;
* (whitespace by default). Every token is spell-corrected independently and
the tokens
* are re-joined with the same delimiter.
*
- * <p>This is the shape produced by OpenNLP tokenizers / token-sample formats
and is
- * what the trainable components consume: a fixed sequence of tokens per
element. Unlike
- * {@link SpellCorrectingObjectStream} in compound mode, this stream is
- * <em>token-count preserving</em> – it never splits or merges tokens,
so the
- * corrected element stays aligned with any parallel annotation (tags,
spans).</p>
+ * <p>This is the shape produced by OpenNLP tokenizers and token-sample
formats, a fixed
+ * sequence of tokens per element. This stream is token-count preserving: it
never splits or
+ * merges tokens, so the corrected element stays aligned with any parallel
annotation.</p>
*
* <p>Correction always runs in
* {@link SpellCheckingCharSequenceNormalizer.Mode#PER_TOKEN per-token} mode
and reuses
@@ -54,7 +50,6 @@ public class SpellCorrectingTokenStream extends
FilterObjectStream<String, Strin
private final SpellCheckingCharSequenceNormalizer normalizer;
private final String delimiter;
- private final Pattern splitPattern;
/**
* Wraps {@code samples} with a default corrector ({@link #DEFAULT_DELIMITER
space}
@@ -62,11 +57,12 @@ public class SpellCorrectingTokenStream extends
FilterObjectStream<String, Strin
*
* @param samples the source token-line stream; must not be {@code null}
* @param spellChecker the engine used to correct tokens; must not be {@code
null}
+ * @throws IllegalArgumentException if {@code samples} or {@code
spellChecker} is {@code null}
*/
public SpellCorrectingTokenStream(ObjectStream<String> samples, SpellChecker
spellChecker) {
this(samples,
SpellCheckingCharSequenceNormalizer.builder(
- Objects.requireNonNull(spellChecker, "spellChecker must not be
null"))
+ requireNonNullArg(spellChecker, "spellChecker"))
.mode(SpellCheckingCharSequenceNormalizer.Mode.PER_TOKEN).build(),
DEFAULT_DELIMITER);
}
@@ -77,9 +73,10 @@ public class SpellCorrectingTokenStream extends
FilterObjectStream<String, Strin
*
* @param samples the source token-line stream; must not be {@code null}
* @param model the loaded model whose engine is used; must not be {@code
null}
+ * @throws IllegalArgumentException if {@code samples} or {@code model} is
{@code null}
*/
public SpellCorrectingTokenStream(ObjectStream<String> samples,
SymSpellModel model) {
- this(samples, Objects.requireNonNull(model, "model must not be
null").getSymSpell());
+ this(samples, requireNonNullArg(model, "model").getSymSpell());
}
/**
@@ -93,23 +90,43 @@ public class SpellCorrectingTokenStream extends
FilterObjectStream<String, Strin
* {@code null}
* @param delimiter the literal token delimiter to split and re-join on;
must not be
* {@code null} or empty
- * @throws NullPointerException if {@code normalizer} or {@code
delimiter} is
- * {@code null}
- * @throws IllegalArgumentException if {@code delimiter} is empty
+ * @throws IllegalArgumentException if {@code samples}, {@code normalizer} or
+ * {@code delimiter} is {@code null}, or if
+ * {@code delimiter} is empty
*/
public SpellCorrectingTokenStream(ObjectStream<String> samples,
SpellCheckingCharSequenceNormalizer
normalizer,
String delimiter) {
- super(samples);
- this.normalizer = Objects.requireNonNull(normalizer, "normalizer must not
be null");
- Objects.requireNonNull(delimiter, "delimiter must not be null");
+ super(requireNonNullArg(samples, "samples"));
+ if (normalizer == null) {
+ throw new IllegalArgumentException("normalizer must not be null");
+ }
+ if (delimiter == null) {
+ throw new IllegalArgumentException("delimiter must not be null");
+ }
if (delimiter.isEmpty()) {
throw new IllegalArgumentException("delimiter must not be empty");
}
+ this.normalizer = normalizer;
this.delimiter = delimiter;
- this.splitPattern = Pattern.compile(Pattern.quote(delimiter));
}
+ /**
+ * Validates that a constructor argument is not {@code null}.
+ *
+ * @param value the argument to check
+ * @param name the parameter name used in the error message
+ * @return {@code value}, never {@code null}
+ * @throws IllegalArgumentException if {@code value} is {@code null}
+ */
+ private static <T> T requireNonNullArg(T value, String name) {
+ if (value == null) {
+ throw new IllegalArgumentException(name + " must not be null");
+ }
+ return value;
+ }
+
+ /** {@inheritDoc} */
@Override
public String read() throws IOException {
final String line = samples.read();
@@ -119,15 +136,19 @@ public class SpellCorrectingTokenStream extends
FilterObjectStream<String, Strin
if (line.isEmpty()) {
return line;
}
- final String[] tokens = splitPattern.split(line, -1);
+ // Split on the literal delimiter and re-join, correcting each non-empty
token.
final StringBuilder out = new StringBuilder(line.length());
- for (int i = 0; i < tokens.length; i++) {
- if (i > 0) {
- out.append(delimiter);
- }
- final String token = tokens[i];
+ int from = 0;
+ while (true) {
+ final int at = line.indexOf(delimiter, from);
+ final String token = at >= 0 ? line.substring(from, at) :
line.substring(from);
// Empty tokens (e.g. from leading/trailing/duplicate delimiters) pass
through.
out.append(token.isEmpty() ? token : normalizer.normalize(token));
+ if (at < 0) {
+ break;
+ }
+ out.append(delimiter);
+ from = at + delimiter.length();
}
return out.toString();
}
diff --git
a/opennlp-extensions/opennlp-spellcheck/src/test/java/opennlp/spellcheck/normalizer/SpellCheckingCharSequenceNormalizerTest.java
b/opennlp-extensions/opennlp-spellcheck/src/test/java/opennlp/spellcheck/normalizer/SpellCheckingCharSequenceNormalizerTest.java
index 7e003c894..5c8f5c5e5 100644
---
a/opennlp-extensions/opennlp-spellcheck/src/test/java/opennlp/spellcheck/normalizer/SpellCheckingCharSequenceNormalizerTest.java
+++
b/opennlp-extensions/opennlp-spellcheck/src/test/java/opennlp/spellcheck/normalizer/SpellCheckingCharSequenceNormalizerTest.java
@@ -22,6 +22,7 @@ import java.io.ByteArrayOutputStream;
import java.io.ObjectInputStream;
import java.io.ObjectOutputStream;
+import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
@@ -48,6 +49,13 @@ public class SpellCheckingCharSequenceNormalizerTest {
return n.normalize(text).toString();
}
+ @Test
+ void nullTextThrowsIllegalArgumentException() {
+ // The CharSequenceNormalizer contract: null is rejected, not passed
through.
+ final var normalizer = new SpellCheckingCharSequenceNormalizer(symSpell);
+ assertThrows(IllegalArgumentException.class, () ->
normalizer.normalize(null));
+ }
+
@Test
void perTokenCorrectsTypos() {
// Lower the min length so the 3-letter typos ("teh", "fxo") are also
corrected.
@@ -112,6 +120,29 @@ public class SpellCheckingCharSequenceNormalizerTest {
assertEquals("[email protected]", norm(normalizer, "[email protected]"));
}
+ @Test
+ void punctuationPeelingFollowsUnicodeLetterAndNumberCategories() {
+ // Characterization for the token guards: the peeling classes are the
complement of
+ // \p{L} and \p{N}, so Unicode punctuation peels like ASCII, and a
combining mark
+ // (category Mark) peels off as a suffix rather than staying attached to
the core.
+ final var normalizer = new SpellCheckingCharSequenceNormalizer(symSpell);
+ assertEquals("\u00ABquick\u00BB", norm(normalizer, "\u00ABquikc\u00BB"));
+ assertEquals("quick\u0301", norm(normalizer, "quikc\u0301"));
+ assertEquals("!!!", norm(normalizer, "!!!"));
+ }
+
+ @Test
+ void numberAndUrlGuardsHoldAtTheirExactBoundaries() {
+ final var normalizer = new SpellCheckingCharSequenceNormalizer(symSpell);
+ // Number-like: optional sign, digits with grouping/decimal marks,
optional percent.
+ assertEquals("+3,14%", norm(normalizer, "+3,14%"));
+ assertEquals("-42%", norm(normalizer, "-42%"));
+ assertEquals("1.2.3", norm(normalizer, "1.2.3"));
+ // URL-like third alternative: a bare domain with a known TLD is never
corrected.
+ assertEquals("quikc.com", norm(normalizer, "quikc.com"));
+ assertEquals("www.quikc.com/broen", norm(normalizer,
"www.quikc.com/broen"));
+ }
+
@Test
void compoundModeRepairsSpaceMerges() {
final var normalizer =
SpellCheckingCharSequenceNormalizer.builder(symSpell)
@@ -140,10 +171,9 @@ public class SpellCheckingCharSequenceNormalizerTest {
}
@Test
- void emptyAndNullInputsArePassedThrough() {
+ void emptyInputIsPassedThrough() {
final var normalizer = new SpellCheckingCharSequenceNormalizer(symSpell);
assertEquals("", norm(normalizer, ""));
- org.junit.jupiter.api.Assertions.assertNull(normalizer.normalize(null));
}
@Test
@@ -178,7 +208,45 @@ public class SpellCheckingCharSequenceNormalizerTest {
@Test
void nullCheckerIsRejected() {
- assertThrows(NullPointerException.class,
+ assertThrows(IllegalArgumentException.class,
() -> new SpellCheckingCharSequenceNormalizer((SymSpell) null));
}
+ private boolean numberLike(String core) {
+ return new
SpellCheckingCharSequenceNormalizer(symSpell).isNumberLike(core);
+ }
+
+ @Test
+ void numberLikeRejectsShapesTheFormerRegexRejected() {
+ // Reject-side pins for the scan structure: at most one trailing percent,
a digit required,
+ // no letters, no bare sign.
+ Assertions.assertFalse(numberLike("5%%"));
+ Assertions.assertFalse(numberLike("+%"));
+ Assertions.assertFalse(numberLike("1,2a"));
+ Assertions.assertFalse(numberLike("+"));
+ Assertions.assertFalse(numberLike(""));
+ Assertions.assertFalse(numberLike("%"));
+ Assertions.assertFalse(numberLike("..,,"));
+ Assertions.assertTrue(numberLike("+3,14%"));
+ Assertions.assertTrue(numberLike("5%"));
+ }
+
+ @Test
+ void numberLikeMatchesTheFormerRegexOverGeneratedTokens() {
+ // Differential over the token alphabet of the former
"[+-]?[\\d.,]*\\d[\\d.,]*%?" guard.
+ final java.util.regex.Pattern former =
+ java.util.regex.Pattern.compile("[+-]?[\\d.,]*\\d[\\d.,]*%?");
+ final char[] alphabet = {'+', '-', '%', '.', ',', '0', '5', '9', 'a'};
+ final java.util.Random random = new java.util.Random(42);
+ for (int round = 0; round < 20_000; round++) {
+ final int length = random.nextInt(7);
+ final StringBuilder token = new StringBuilder();
+ for (int i = 0; i < length; i++) {
+ token.append(alphabet[random.nextInt(alphabet.length)]);
+ }
+ final String core = token.toString();
+ Assertions.assertEquals(former.matcher(core).matches(),
+ numberLike(core),
+ () -> "core: " + core);
+ }
+ }
}
diff --git
a/opennlp-extensions/opennlp-spellcheck/src/test/java/opennlp/spellcheck/stream/SpellCorrectingObjectStreamTest.java
b/opennlp-extensions/opennlp-spellcheck/src/test/java/opennlp/spellcheck/stream/SpellCorrectingObjectStreamTest.java
index 8d5ac224b..35204a2eb 100644
---
a/opennlp-extensions/opennlp-spellcheck/src/test/java/opennlp/spellcheck/stream/SpellCorrectingObjectStreamTest.java
+++
b/opennlp-extensions/opennlp-spellcheck/src/test/java/opennlp/spellcheck/stream/SpellCorrectingObjectStreamTest.java
@@ -24,6 +24,9 @@ import java.nio.charset.StandardCharsets;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
+import opennlp.spellcheck.SpellChecker;
+import opennlp.spellcheck.dictionary.SymSpellModel;
+import opennlp.spellcheck.normalizer.SpellCheckingCharSequenceNormalizer;
import opennlp.spellcheck.symspell.SymSpell;
import opennlp.spellcheck.symspell.TinyDictionary;
import opennlp.tools.util.InputStreamFactory;
@@ -103,8 +106,17 @@ public class SpellCorrectingObjectStreamTest {
}
@Test
- void wrapsWithModelConstructorAndNullChecks() {
- assertThrows(NullPointerException.class,
+ void nullArgumentsAreRejected() throws IOException {
+ assertThrows(IllegalArgumentException.class,
() -> new SpellCorrectingObjectStream(null, symSpell));
+ try (ObjectStream<String> samples = lineStream("a\n")) {
+ assertThrows(IllegalArgumentException.class,
+ () -> new SpellCorrectingObjectStream(samples, (SpellChecker) null));
+ assertThrows(IllegalArgumentException.class,
+ () -> new SpellCorrectingObjectStream(samples, (SymSpellModel)
null));
+ assertThrows(IllegalArgumentException.class,
+ () -> new SpellCorrectingObjectStream(samples,
+ (SpellCheckingCharSequenceNormalizer) null));
+ }
}
}
diff --git
a/opennlp-extensions/opennlp-spellcheck/src/test/java/opennlp/spellcheck/stream/SpellCorrectingTokenStreamTest.java
b/opennlp-extensions/opennlp-spellcheck/src/test/java/opennlp/spellcheck/stream/SpellCorrectingTokenStreamTest.java
index 8a7b231fa..97c0fc292 100644
---
a/opennlp-extensions/opennlp-spellcheck/src/test/java/opennlp/spellcheck/stream/SpellCorrectingTokenStreamTest.java
+++
b/opennlp-extensions/opennlp-spellcheck/src/test/java/opennlp/spellcheck/stream/SpellCorrectingTokenStreamTest.java
@@ -25,6 +25,8 @@ import java.util.List;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
+import opennlp.spellcheck.SpellChecker;
+import opennlp.spellcheck.dictionary.SymSpellModel;
import opennlp.spellcheck.normalizer.SpellCheckingCharSequenceNormalizer;
import opennlp.spellcheck.symspell.SymSpell;
import opennlp.spellcheck.symspell.TinyDictionary;
@@ -32,6 +34,7 @@ import opennlp.tools.util.ObjectStream;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
public class SpellCorrectingTokenStreamTest {
@@ -125,4 +128,48 @@ public class SpellCorrectingTokenStreamTest {
assertEquals("world fox", stream.read());
}
}
+ @Test
+ void leadingAndTrailingDelimitersPreserveEmptyEdgeTokens() throws
IOException {
+ // split(line, -1) kept empty edge tokens; the indexOf walk must re-join
them unchanged.
+ final var normalizer =
SpellCheckingCharSequenceNormalizer.builder(symSpell)
+ .minTokenLength(3).build();
+ try (ObjectStream<String> stream = new SpellCorrectingTokenStream(
+ new ListStream(" teh quikc "), normalizer, " ")) {
+ assertEquals(" the quick ", stream.read());
+ }
+ }
+
+ @Test
+ void multiCharacterDelimiterSplitsAndRejoinsLiterally() throws IOException {
+ final var normalizer =
SpellCheckingCharSequenceNormalizer.builder(symSpell)
+ .minTokenLength(3).build();
+ try (ObjectStream<String> stream = new SpellCorrectingTokenStream(
+ new ListStream("teh::quikc::broen"), normalizer, "::")) {
+ assertEquals("the::quick::brown", stream.read());
+ }
+ }
+
+ @Test
+ void duplicateDelimitersAtTheEdgesSurviveTheRoundTrip() throws IOException {
+ final var normalizer =
SpellCheckingCharSequenceNormalizer.builder(symSpell)
+ .minTokenLength(3).build();
+ try (ObjectStream<String> stream = new SpellCorrectingTokenStream(
+ new ListStream("--teh--broen--"), normalizer, "--")) {
+ assertEquals("--the--brown--", stream.read());
+ }
+ }
+
+ @Test
+ void nullArgumentsAreRejected() {
+ final var normalizer =
SpellCheckingCharSequenceNormalizer.builder(symSpell)
+ .mode(SpellCheckingCharSequenceNormalizer.Mode.PER_TOKEN).build();
+ assertThrows(IllegalArgumentException.class,
+ () -> new SpellCorrectingTokenStream(null, symSpell));
+ assertThrows(IllegalArgumentException.class,
+ () -> new SpellCorrectingTokenStream(new ListStream("a"),
(SpellChecker) null));
+ assertThrows(IllegalArgumentException.class,
+ () -> new SpellCorrectingTokenStream(new ListStream("a"),
(SymSpellModel) null));
+ assertThrows(IllegalArgumentException.class,
+ () -> new SpellCorrectingTokenStream(null, normalizer, " "));
+ }
}
diff --git
a/opennlp-tools/src/test/java/opennlp/tools/util/normalizer/TwitterCharSequenceNormalizerTest.java
b/opennlp-tools/src/test/java/opennlp/tools/util/normalizer/TwitterCharSequenceNormalizerTest.java
deleted file mode 100644
index 528af33a4..000000000
---
a/opennlp-tools/src/test/java/opennlp/tools/util/normalizer/TwitterCharSequenceNormalizerTest.java
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package opennlp.tools.util.normalizer;
-
-import org.junit.jupiter.api.Assertions;
-import org.junit.jupiter.api.Test;
-
-
-public class TwitterCharSequenceNormalizerTest {
-
- private final TwitterCharSequenceNormalizer normalizer =
TwitterCharSequenceNormalizer.getInstance();
-
- @Test
- void normalizeHashtag() {
- Assertions.assertEquals("asdf 2nnfdf", normalizer.normalize("asdf
#hasdk23 2nnfdf"));
- }
-
- @Test
- void normalizeUser() {
- Assertions.assertEquals("asdf 2nnfdf", normalizer.normalize("asdf
@hasdk23 2nnfdf"));
- }
-
- @Test
- void normalizeRT() {
- Assertions.assertEquals(" 2nnfdf", normalizer.normalize("RT RT RT
2nnfdf"));
- }
-
- @Test
- void normalizeLaugh() {
- Assertions.assertEquals("ahahah", normalizer.normalize("ahahahah"));
- Assertions.assertEquals("haha", normalizer.normalize("hahha"));
- Assertions.assertEquals("haha", normalizer.normalize("hahaa"));
- Assertions.assertEquals("ahaha",
normalizer.normalize("ahahahahhahahhahahaaaa"));
- Assertions.assertEquals("jaja", normalizer.normalize("jajjajajaja"));
- }
-
-
- @Test
- void normalizeFace() {
- Assertions.assertEquals("hello hello", normalizer.normalize("hello :-)
hello"));
- Assertions.assertEquals("hello hello", normalizer.normalize("hello ;)
hello"));
- Assertions.assertEquals(" hello", normalizer.normalize(":) hello"));
- Assertions.assertEquals("hello ", normalizer.normalize("hello :P"));
- }
-
-}
