krickert commented on code in PR #1151:
URL: https://github.com/apache/opennlp/pull/1151#discussion_r3564894197
##########
opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/UrlCharSequenceNormalizer.java:
##########
@@ -16,29 +16,174 @@
*/
package opennlp.tools.util.normalizer;
-import java.util.regex.Pattern;
-
/**
- * A {@link UrlCharSequenceNormalizer} implementation that normalizes text
+ * A {@link CharSequenceNormalizer} implementation that normalizes text
* in terms of URls and email addresses. Every encounter will be replaced by a
whitespace.
+ *
+ * <p>Two forward cursor passes reproduce, byte for byte, the accept/reject
boundary of the
+ * former regex implementation:</p>
+ * <ol>
+ * <li>URLs: a lowercase {@code http://} or {@code https://} scheme followed
by at least one
+ * character out of the former body class {@code
[-_.?&~;+=/#0-9A-Za-z]}; the match ends at
+ * the first character outside that class, so a colon (port), percent
escape, at sign
+ * (userinfo), or non-ASCII label cuts it short, exactly as before.</li>
+ * <li>Email addresses: a maximal run of the former local-part class {@code
[-+_.0-9A-Za-z]}
+ * whose left neighbor is outside that class (the lookbehind), an {@code
@}, and then a
+ * maximal run of the former domain class {@code [-.0-9A-Za-z]} that
must not start with a
+ * dot and must span at least two chars. The two-char minimum falls out
of the former
+ * {@code [-0-9A-Za-z]+[-.0-9A-Za-z]+} pair: with no dot following, the
first class lent
+ * its last character to the second through backtracking, which needed a
run of two.</li>
+ * </ol>
*/
public class UrlCharSequenceNormalizer implements CharSequenceNormalizer {
private static final long serialVersionUID = 2023145028634552389L;
- private static final Pattern URL_REGEX =
- Pattern.compile("https?://[-_.?&~;+=/#0-9A-Za-z]+");
- private static final Pattern MAIL_REGEX =
-
Pattern.compile("(?<![-+_.0-9A-Za-z])[-+_.0-9A-Za-z]+@[-0-9A-Za-z]+[-.0-9A-Za-z]+");
+
+ private static final CodePointSet ASCII_ALNUM = CodePointSet.ofRange('0',
'9')
+ .union(CodePointSet.ofRange('A', 'Z'))
+ .union(CodePointSet.ofRange('a', 'z'));
+
+ // The former URL body class: [-_.?&~;+=/#0-9A-Za-z]
+ private static final CodePointSet URL_BODY =
+ ASCII_ALNUM.union(CodePointSet.of('-', '_', '.', '?', '&', '~', ';',
'+', '=', '/', '#'));
+
+ // The former mail local-part class (also the lookbehind class):
[-+_.0-9A-Za-z]
+ private static final CodePointSet MAIL_LOCAL =
+ ASCII_ALNUM.union(CodePointSet.of('-', '+', '_', '.'));
+
+ // The former first domain class: [-0-9A-Za-z]
+ private static final CodePointSet MAIL_DOMAIN_START =
ASCII_ALNUM.union(CodePointSet.of('-'));
+
+ // The former second domain class: [-.0-9A-Za-z]
+ private static final CodePointSet MAIL_DOMAIN =
MAIL_DOMAIN_START.union(CodePointSet.of('.'));
private static final UrlCharSequenceNormalizer INSTANCE = new
UrlCharSequenceNormalizer();
public static UrlCharSequenceNormalizer getInstance() {
return INSTANCE;
}
+ /**
+ * @throws IllegalArgumentException Thrown if {@code text} is {@code null}.
+ */
@Override
- public CharSequence normalize (CharSequence text) {
- String modified = URL_REGEX.matcher(text).replaceAll(" ");
- return MAIL_REGEX.matcher(modified).replaceAll(" ");
+ public CharSequence normalize(CharSequence text) {
+ if (text == null) {
+ throw new IllegalArgumentException("The text must not be null.");
+ }
+ return removeMailAddresses(removeUrls(text));
+ }
+
+ // "https?://[-_.?&~;+=/#0-9A-Za-z]+" -> " "
+ private static CharSequence removeUrls(CharSequence text) {
+ final int length = text.length();
+ StringBuilder out = null;
+ int i = 0;
+ while (i < length) {
+ final int end = matchUrlEnd(text, i);
+ if (end > i) {
+ if (out == null) {
+ out = new StringBuilder(length).append(text, 0, i);
+ }
+ out.append(' ');
+ i = end;
+ } else {
+ if (out != null) {
+ out.append(text.charAt(i));
+ }
+ i++;
+ }
+ }
+ return out == null ? text : out.toString();
+ }
+
+ // Returns the exclusive end of a URL match starting at start, or -1 if
there is none.
+ private static int matchUrlEnd(CharSequence text, int start) {
Review Comment:
Done.
##########
opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/UrlCharSequenceNormalizer.java:
##########
@@ -16,29 +16,174 @@
*/
package opennlp.tools.util.normalizer;
-import java.util.regex.Pattern;
-
/**
- * A {@link UrlCharSequenceNormalizer} implementation that normalizes text
+ * A {@link CharSequenceNormalizer} implementation that normalizes text
* in terms of URls and email addresses. Every encounter will be replaced by a
whitespace.
+ *
+ * <p>Two forward cursor passes reproduce, byte for byte, the accept/reject
boundary of the
+ * former regex implementation:</p>
+ * <ol>
+ * <li>URLs: a lowercase {@code http://} or {@code https://} scheme followed
by at least one
+ * character out of the former body class {@code
[-_.?&~;+=/#0-9A-Za-z]}; the match ends at
+ * the first character outside that class, so a colon (port), percent
escape, at sign
+ * (userinfo), or non-ASCII label cuts it short, exactly as before.</li>
+ * <li>Email addresses: a maximal run of the former local-part class {@code
[-+_.0-9A-Za-z]}
+ * whose left neighbor is outside that class (the lookbehind), an {@code
@}, and then a
+ * maximal run of the former domain class {@code [-.0-9A-Za-z]} that
must not start with a
+ * dot and must span at least two chars. The two-char minimum falls out
of the former
+ * {@code [-0-9A-Za-z]+[-.0-9A-Za-z]+} pair: with no dot following, the
first class lent
+ * its last character to the second through backtracking, which needed a
run of two.</li>
+ * </ol>
*/
public class UrlCharSequenceNormalizer implements CharSequenceNormalizer {
private static final long serialVersionUID = 2023145028634552389L;
- private static final Pattern URL_REGEX =
- Pattern.compile("https?://[-_.?&~;+=/#0-9A-Za-z]+");
- private static final Pattern MAIL_REGEX =
-
Pattern.compile("(?<![-+_.0-9A-Za-z])[-+_.0-9A-Za-z]+@[-0-9A-Za-z]+[-.0-9A-Za-z]+");
+
+ private static final CodePointSet ASCII_ALNUM = CodePointSet.ofRange('0',
'9')
+ .union(CodePointSet.ofRange('A', 'Z'))
+ .union(CodePointSet.ofRange('a', 'z'));
+
+ // The former URL body class: [-_.?&~;+=/#0-9A-Za-z]
+ private static final CodePointSet URL_BODY =
+ ASCII_ALNUM.union(CodePointSet.of('-', '_', '.', '?', '&', '~', ';',
'+', '=', '/', '#'));
+
+ // The former mail local-part class (also the lookbehind class):
[-+_.0-9A-Za-z]
+ private static final CodePointSet MAIL_LOCAL =
+ ASCII_ALNUM.union(CodePointSet.of('-', '+', '_', '.'));
+
+ // The former first domain class: [-0-9A-Za-z]
+ private static final CodePointSet MAIL_DOMAIN_START =
ASCII_ALNUM.union(CodePointSet.of('-'));
+
+ // The former second domain class: [-.0-9A-Za-z]
+ private static final CodePointSet MAIL_DOMAIN =
MAIL_DOMAIN_START.union(CodePointSet.of('.'));
private static final UrlCharSequenceNormalizer INSTANCE = new
UrlCharSequenceNormalizer();
public static UrlCharSequenceNormalizer getInstance() {
return INSTANCE;
}
+ /**
+ * @throws IllegalArgumentException Thrown if {@code text} is {@code null}.
+ */
@Override
- public CharSequence normalize (CharSequence text) {
- String modified = URL_REGEX.matcher(text).replaceAll(" ");
- return MAIL_REGEX.matcher(modified).replaceAll(" ");
+ public CharSequence normalize(CharSequence text) {
+ if (text == null) {
+ throw new IllegalArgumentException("The text must not be null.");
+ }
+ return removeMailAddresses(removeUrls(text));
+ }
+
+ // "https?://[-_.?&~;+=/#0-9A-Za-z]+" -> " "
+ private static CharSequence removeUrls(CharSequence text) {
+ final int length = text.length();
+ StringBuilder out = null;
+ int i = 0;
+ while (i < length) {
+ final int end = matchUrlEnd(text, i);
+ if (end > i) {
+ if (out == null) {
+ out = new StringBuilder(length).append(text, 0, i);
+ }
+ out.append(' ');
+ i = end;
+ } else {
+ if (out != null) {
+ out.append(text.charAt(i));
+ }
+ i++;
+ }
+ }
+ return out == null ? text : out.toString();
+ }
+
+ // Returns the exclusive end of a URL match starting at start, or -1 if
there is none.
+ private static int matchUrlEnd(CharSequence text, int start) {
+ final int length = text.length();
+ if (!regionEquals(text, start, "http")) {
+ return -1;
+ }
+ int bodyStart = start + 4;
+ if (bodyStart < length && text.charAt(bodyStart) == 's'
+ && regionEquals(text, bodyStart + 1, "://")) {
+ bodyStart += 4;
+ } else if (regionEquals(text, bodyStart, "://")) {
+ bodyStart += 3;
+ } else {
+ return -1;
+ }
+ if (bodyStart >= length || !URL_BODY.contains(text.charAt(bodyStart))) {
+ return -1;
+ }
+ int end = bodyStart + 1;
+ while (end < length && URL_BODY.contains(text.charAt(end))) {
+ end++;
+ }
+ return end;
+ }
+
+ private static boolean regionEquals(CharSequence text, int at, String
literal) {
Review Comment:
Done.
##########
opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/UrlCharSequenceNormalizer.java:
##########
@@ -16,29 +16,174 @@
*/
package opennlp.tools.util.normalizer;
-import java.util.regex.Pattern;
-
/**
- * A {@link UrlCharSequenceNormalizer} implementation that normalizes text
+ * A {@link CharSequenceNormalizer} implementation that normalizes text
* in terms of URls and email addresses. Every encounter will be replaced by a
whitespace.
+ *
+ * <p>Two forward cursor passes reproduce, byte for byte, the accept/reject
boundary of the
+ * former regex implementation:</p>
+ * <ol>
+ * <li>URLs: a lowercase {@code http://} or {@code https://} scheme followed
by at least one
+ * character out of the former body class {@code
[-_.?&~;+=/#0-9A-Za-z]}; the match ends at
+ * the first character outside that class, so a colon (port), percent
escape, at sign
+ * (userinfo), or non-ASCII label cuts it short, exactly as before.</li>
+ * <li>Email addresses: a maximal run of the former local-part class {@code
[-+_.0-9A-Za-z]}
+ * whose left neighbor is outside that class (the lookbehind), an {@code
@}, and then a
+ * maximal run of the former domain class {@code [-.0-9A-Za-z]} that
must not start with a
+ * dot and must span at least two chars. The two-char minimum falls out
of the former
+ * {@code [-0-9A-Za-z]+[-.0-9A-Za-z]+} pair: with no dot following, the
first class lent
+ * its last character to the second through backtracking, which needed a
run of two.</li>
+ * </ol>
*/
public class UrlCharSequenceNormalizer implements CharSequenceNormalizer {
private static final long serialVersionUID = 2023145028634552389L;
- private static final Pattern URL_REGEX =
- Pattern.compile("https?://[-_.?&~;+=/#0-9A-Za-z]+");
- private static final Pattern MAIL_REGEX =
-
Pattern.compile("(?<![-+_.0-9A-Za-z])[-+_.0-9A-Za-z]+@[-0-9A-Za-z]+[-.0-9A-Za-z]+");
+
+ private static final CodePointSet ASCII_ALNUM = CodePointSet.ofRange('0',
'9')
+ .union(CodePointSet.ofRange('A', 'Z'))
+ .union(CodePointSet.ofRange('a', 'z'));
+
+ // The former URL body class: [-_.?&~;+=/#0-9A-Za-z]
+ private static final CodePointSet URL_BODY =
+ ASCII_ALNUM.union(CodePointSet.of('-', '_', '.', '?', '&', '~', ';',
'+', '=', '/', '#'));
+
+ // The former mail local-part class (also the lookbehind class):
[-+_.0-9A-Za-z]
+ private static final CodePointSet MAIL_LOCAL =
+ ASCII_ALNUM.union(CodePointSet.of('-', '+', '_', '.'));
+
+ // The former first domain class: [-0-9A-Za-z]
+ private static final CodePointSet MAIL_DOMAIN_START =
ASCII_ALNUM.union(CodePointSet.of('-'));
+
+ // The former second domain class: [-.0-9A-Za-z]
+ private static final CodePointSet MAIL_DOMAIN =
MAIL_DOMAIN_START.union(CodePointSet.of('.'));
private static final UrlCharSequenceNormalizer INSTANCE = new
UrlCharSequenceNormalizer();
public static UrlCharSequenceNormalizer getInstance() {
return INSTANCE;
}
+ /**
+ * @throws IllegalArgumentException Thrown if {@code text} is {@code null}.
+ */
@Override
- public CharSequence normalize (CharSequence text) {
- String modified = URL_REGEX.matcher(text).replaceAll(" ");
- return MAIL_REGEX.matcher(modified).replaceAll(" ");
+ public CharSequence normalize(CharSequence text) {
+ if (text == null) {
+ throw new IllegalArgumentException("The text must not be null.");
+ }
+ return removeMailAddresses(removeUrls(text));
+ }
+
+ // "https?://[-_.?&~;+=/#0-9A-Za-z]+" -> " "
+ private static CharSequence removeUrls(CharSequence text) {
+ final int length = text.length();
+ StringBuilder out = null;
+ int i = 0;
+ while (i < length) {
+ final int end = matchUrlEnd(text, i);
+ if (end > i) {
+ if (out == null) {
+ out = new StringBuilder(length).append(text, 0, i);
+ }
+ out.append(' ');
+ i = end;
+ } else {
+ if (out != null) {
+ out.append(text.charAt(i));
+ }
+ i++;
+ }
+ }
+ return out == null ? text : out.toString();
+ }
+
+ // Returns the exclusive end of a URL match starting at start, or -1 if
there is none.
+ private static int matchUrlEnd(CharSequence text, int start) {
+ final int length = text.length();
+ if (!regionEquals(text, start, "http")) {
+ return -1;
+ }
+ int bodyStart = start + 4;
+ if (bodyStart < length && text.charAt(bodyStart) == 's'
+ && regionEquals(text, bodyStart + 1, "://")) {
+ bodyStart += 4;
+ } else if (regionEquals(text, bodyStart, "://")) {
+ bodyStart += 3;
+ } else {
+ return -1;
+ }
+ if (bodyStart >= length || !URL_BODY.contains(text.charAt(bodyStart))) {
+ return -1;
+ }
+ int end = bodyStart + 1;
+ while (end < length && URL_BODY.contains(text.charAt(end))) {
+ end++;
+ }
+ return end;
+ }
+
+ private static boolean regionEquals(CharSequence text, int at, String
literal) {
+ if (at + literal.length() > text.length()) {
+ return false;
+ }
+ for (int k = 0; k < literal.length(); k++) {
+ if (text.charAt(at + k) != literal.charAt(k)) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ // "(?<![-+_.0-9A-Za-z])[-+_.0-9A-Za-z]+@[-0-9A-Za-z]+[-.0-9A-Za-z]+" -> " "
+ private static CharSequence removeMailAddresses(CharSequence text) {
Review Comment:
Done.
##########
opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/UrlCharSequenceNormalizer.java:
##########
@@ -16,29 +16,174 @@
*/
package opennlp.tools.util.normalizer;
-import java.util.regex.Pattern;
-
/**
- * A {@link UrlCharSequenceNormalizer} implementation that normalizes text
+ * A {@link CharSequenceNormalizer} implementation that normalizes text
* in terms of URls and email addresses. Every encounter will be replaced by a
whitespace.
+ *
+ * <p>Two forward cursor passes reproduce, byte for byte, the accept/reject
boundary of the
+ * former regex implementation:</p>
+ * <ol>
+ * <li>URLs: a lowercase {@code http://} or {@code https://} scheme followed
by at least one
+ * character out of the former body class {@code
[-_.?&~;+=/#0-9A-Za-z]}; the match ends at
+ * the first character outside that class, so a colon (port), percent
escape, at sign
+ * (userinfo), or non-ASCII label cuts it short, exactly as before.</li>
+ * <li>Email addresses: a maximal run of the former local-part class {@code
[-+_.0-9A-Za-z]}
+ * whose left neighbor is outside that class (the lookbehind), an {@code
@}, and then a
+ * maximal run of the former domain class {@code [-.0-9A-Za-z]} that
must not start with a
+ * dot and must span at least two chars. The two-char minimum falls out
of the former
+ * {@code [-0-9A-Za-z]+[-.0-9A-Za-z]+} pair: with no dot following, the
first class lent
+ * its last character to the second through backtracking, which needed a
run of two.</li>
+ * </ol>
*/
public class UrlCharSequenceNormalizer implements CharSequenceNormalizer {
private static final long serialVersionUID = 2023145028634552389L;
- private static final Pattern URL_REGEX =
- Pattern.compile("https?://[-_.?&~;+=/#0-9A-Za-z]+");
- private static final Pattern MAIL_REGEX =
-
Pattern.compile("(?<![-+_.0-9A-Za-z])[-+_.0-9A-Za-z]+@[-0-9A-Za-z]+[-.0-9A-Za-z]+");
+
+ private static final CodePointSet ASCII_ALNUM = CodePointSet.ofRange('0',
'9')
+ .union(CodePointSet.ofRange('A', 'Z'))
+ .union(CodePointSet.ofRange('a', 'z'));
+
+ // The former URL body class: [-_.?&~;+=/#0-9A-Za-z]
+ private static final CodePointSet URL_BODY =
+ ASCII_ALNUM.union(CodePointSet.of('-', '_', '.', '?', '&', '~', ';',
'+', '=', '/', '#'));
+
+ // The former mail local-part class (also the lookbehind class):
[-+_.0-9A-Za-z]
+ private static final CodePointSet MAIL_LOCAL =
+ ASCII_ALNUM.union(CodePointSet.of('-', '+', '_', '.'));
+
+ // The former first domain class: [-0-9A-Za-z]
+ private static final CodePointSet MAIL_DOMAIN_START =
ASCII_ALNUM.union(CodePointSet.of('-'));
+
+ // The former second domain class: [-.0-9A-Za-z]
+ private static final CodePointSet MAIL_DOMAIN =
MAIL_DOMAIN_START.union(CodePointSet.of('.'));
private static final UrlCharSequenceNormalizer INSTANCE = new
UrlCharSequenceNormalizer();
public static UrlCharSequenceNormalizer getInstance() {
return INSTANCE;
}
+ /**
+ * @throws IllegalArgumentException Thrown if {@code text} is {@code null}.
+ */
@Override
- public CharSequence normalize (CharSequence text) {
- String modified = URL_REGEX.matcher(text).replaceAll(" ");
- return MAIL_REGEX.matcher(modified).replaceAll(" ");
+ public CharSequence normalize(CharSequence text) {
+ if (text == null) {
+ throw new IllegalArgumentException("The text must not be null.");
+ }
+ return removeMailAddresses(removeUrls(text));
+ }
+
+ // "https?://[-_.?&~;+=/#0-9A-Za-z]+" -> " "
+ private static CharSequence removeUrls(CharSequence text) {
+ final int length = text.length();
+ StringBuilder out = null;
+ int i = 0;
+ while (i < length) {
+ final int end = matchUrlEnd(text, i);
+ if (end > i) {
+ if (out == null) {
+ out = new StringBuilder(length).append(text, 0, i);
+ }
+ out.append(' ');
+ i = end;
+ } else {
+ if (out != null) {
+ out.append(text.charAt(i));
+ }
+ i++;
+ }
+ }
+ return out == null ? text : out.toString();
+ }
+
+ // Returns the exclusive end of a URL match starting at start, or -1 if
there is none.
+ private static int matchUrlEnd(CharSequence text, int start) {
+ final int length = text.length();
+ if (!regionEquals(text, start, "http")) {
+ return -1;
+ }
+ int bodyStart = start + 4;
+ if (bodyStart < length && text.charAt(bodyStart) == 's'
+ && regionEquals(text, bodyStart + 1, "://")) {
+ bodyStart += 4;
+ } else if (regionEquals(text, bodyStart, "://")) {
+ bodyStart += 3;
+ } else {
+ return -1;
+ }
+ if (bodyStart >= length || !URL_BODY.contains(text.charAt(bodyStart))) {
+ return -1;
+ }
+ int end = bodyStart + 1;
+ while (end < length && URL_BODY.contains(text.charAt(end))) {
+ end++;
+ }
+ return end;
+ }
+
+ private static boolean regionEquals(CharSequence text, int at, String
literal) {
+ if (at + literal.length() > text.length()) {
+ return false;
+ }
+ for (int k = 0; k < literal.length(); k++) {
+ if (text.charAt(at + k) != literal.charAt(k)) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ // "(?<![-+_.0-9A-Za-z])[-+_.0-9A-Za-z]+@[-0-9A-Za-z]+[-.0-9A-Za-z]+" -> " "
+ private static CharSequence removeMailAddresses(CharSequence text) {
+ final int length = text.length();
+ StringBuilder out = null;
+ int i = 0;
+ while (i < length) {
+ final int end = matchMailEnd(text, i);
+ if (end > i) {
+ if (out == null) {
+ out = new StringBuilder(length).append(text, 0, i);
+ }
+ out.append(' ');
+ i = end;
+ } else {
+ if (out != null) {
+ out.append(text.charAt(i));
+ }
+ i++;
+ }
+ }
+ return out == null ? text : out.toString();
+ }
+
+ // Returns the exclusive end of a mail match starting at start, or -1 if
there is none.
+ private static int matchMailEnd(CharSequence text, int start) {
Review Comment:
Done.
##########
opennlp-extensions/opennlp-spellcheck/src/main/java/opennlp/spellcheck/normalizer/SpellCheckingCharSequenceNormalizer.java:
##########
@@ -84,21 +84,18 @@ public class SpellCheckingCharSequenceNormalizer implements
CharSequenceNormaliz
/** The default minimum token length below which tokens are left untouched.
*/
public static final int DEFAULT_MIN_TOKEN_LENGTH = 4;
- /** Matches tokens that are entirely digits, optionally with
grouping/decimal marks. */
- private static final Pattern NUMBER_LIKE =
Pattern.compile("[+-]?[\\d.,]*\\d[\\d.,]*%?");
-
- /** Matches URL- and email-like tokens that should never be spell-corrected.
*/
+ /**
+ * Matches URL- and email-like tokens that should never be spell-corrected.
+ * Deliberately still a regex: the three-alternative shape (scheme or www
prefix, mail
Review Comment:
Done, dropped.
##########
opennlp-extensions/opennlp-spellcheck/src/main/java/opennlp/spellcheck/normalizer/SpellCheckingCharSequenceNormalizer.java:
##########
@@ -320,9 +318,79 @@ private static boolean isAllUpper(String s) {
return sawLetter && s.length() > 1;
}
- private static String match(Pattern pattern, String s) {
- final var m = pattern.matcher(s);
- return m.find() ? m.group() : "";
+ /**
+ * Length of the leading run of non-letter/non-number code points (the former
+ * {@code "^[^\p{L}\p{N}]+"}), found by a forward cursor scan.
+ */
+ private static int leadingNonWordLength(String token) {
Review Comment:
Done.
##########
opennlp-extensions/opennlp-spellcheck/src/main/java/opennlp/spellcheck/normalizer/SpellCheckingCharSequenceNormalizer.java:
##########
@@ -320,9 +318,79 @@ private static boolean isAllUpper(String s) {
return sawLetter && s.length() > 1;
}
- private static String match(Pattern pattern, String s) {
- final var m = pattern.matcher(s);
- return m.find() ? m.group() : "";
+ /**
+ * Length of the leading run of non-letter/non-number code points (the former
+ * {@code "^[^\p{L}\p{N}]+"}), found by a forward cursor scan.
+ */
+ private static int leadingNonWordLength(String token) {
+ int i = 0;
+ while (i < token.length()) {
+ final int codePoint = token.codePointAt(i);
+ if (isLetterOrNumber(codePoint)) {
+ break;
+ }
+ i += Character.charCount(codePoint);
+ }
+ return i;
+ }
+
+ /**
+ * Length of the trailing run of non-letter/non-number code points after
{@code from} (the
+ * former {@code "[^\p{L}\p{N}]+$"} applied behind the peeled prefix), found
by a backward
+ * cursor scan.
+ */
+ private static int trailingNonWordLength(String token, int from) {
Review Comment:
Done.
##########
opennlp-extensions/opennlp-spellcheck/src/main/java/opennlp/spellcheck/normalizer/SpellCheckingCharSequenceNormalizer.java:
##########
@@ -320,9 +318,79 @@ private static boolean isAllUpper(String s) {
return sawLetter && s.length() > 1;
}
- private static String match(Pattern pattern, String s) {
- final var m = pattern.matcher(s);
- return m.find() ? m.group() : "";
+ /**
+ * Length of the leading run of non-letter/non-number code points (the former
+ * {@code "^[^\p{L}\p{N}]+"}), found by a forward cursor scan.
+ */
+ private static int leadingNonWordLength(String token) {
+ int i = 0;
+ while (i < token.length()) {
+ final int codePoint = token.codePointAt(i);
+ if (isLetterOrNumber(codePoint)) {
+ break;
+ }
+ i += Character.charCount(codePoint);
+ }
+ return i;
+ }
+
+ /**
+ * Length of the trailing run of non-letter/non-number code points after
{@code from} (the
+ * former {@code "[^\p{L}\p{N}]+$"} applied behind the peeled prefix), found
by a backward
+ * cursor scan.
+ */
+ private static int trailingNonWordLength(String token, int from) {
+ int end = token.length();
+ while (end > from) {
+ final int codePoint = token.codePointBefore(end);
+ if (isLetterOrNumber(codePoint)) {
+ break;
+ }
+ end -= Character.charCount(codePoint);
+ }
+ return token.length() - end;
+ }
+
+ // \p{L} is Character.isLetter; \p{N} is the Nd, Nl, and No categories (note
that
+ // Character.isDigit only covers Nd).
+ private static boolean isLetterOrNumber(int codePoint) {
Review Comment:
Done.
##########
opennlp-extensions/opennlp-spellcheck/src/main/java/opennlp/spellcheck/normalizer/SpellCheckingCharSequenceNormalizer.java:
##########
@@ -320,9 +318,79 @@ private static boolean isAllUpper(String s) {
return sawLetter && s.length() > 1;
}
- private static String match(Pattern pattern, String s) {
- final var m = pattern.matcher(s);
- return m.find() ? m.group() : "";
+ /**
+ * Length of the leading run of non-letter/non-number code points (the former
+ * {@code "^[^\p{L}\p{N}]+"}), found by a forward cursor scan.
+ */
+ private static int leadingNonWordLength(String token) {
+ int i = 0;
+ while (i < token.length()) {
+ final int codePoint = token.codePointAt(i);
+ if (isLetterOrNumber(codePoint)) {
+ break;
+ }
+ i += Character.charCount(codePoint);
+ }
+ return i;
+ }
+
+ /**
+ * Length of the trailing run of non-letter/non-number code points after
{@code from} (the
+ * former {@code "[^\p{L}\p{N}]+$"} applied behind the peeled prefix), found
by a backward
+ * cursor scan.
+ */
+ private static int trailingNonWordLength(String token, int from) {
+ int end = token.length();
+ while (end > from) {
+ final int codePoint = token.codePointBefore(end);
+ if (isLetterOrNumber(codePoint)) {
+ break;
+ }
+ end -= Character.charCount(codePoint);
+ }
+ return token.length() - end;
+ }
+
+ // \p{L} is Character.isLetter; \p{N} is the Nd, Nl, and No categories (note
that
+ // Character.isDigit only covers Nd).
+ private static boolean isLetterOrNumber(int codePoint) {
+ if (Character.isLetter(codePoint)) {
+ return true;
+ }
+ final int type = Character.getType(codePoint);
+ return type == Character.DECIMAL_DIGIT_NUMBER
+ || type == Character.LETTER_NUMBER
+ || type == Character.OTHER_NUMBER;
+ }
+
+ /**
+ * A cursor-scan replacement for the former {@code
"[+-]?[\d.,]*\d[\d.,]*%?"} full-token
+ * match: an optional sign, then digits with optional grouping/decimal marks
containing at
+ * least one ASCII digit, then an optional trailing percent sign.
Package-private so the tests
+ * can hold the classification against the former regex directly.
+ */
+ static boolean isNumberLike(String core) {
Review Comment:
static dropped; kept package private so the differential test can drive the
classification directly, and the javadoc now says so.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
