krickert commented on code in PR #1151:
URL: https://github.com/apache/opennlp/pull/1151#discussion_r3564893700
##########
opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/TwitterCharSequenceNormalizer.java:
##########
@@ -16,40 +16,263 @@
*/
package opennlp.tools.util.normalizer;
-import java.util.regex.Pattern;
-
/**
- * A {@link TwitterCharSequenceNormalizer} implementation that normalizes text
+ * A {@link CharSequenceNormalizer} implementation that normalizes text
* in terms of Twitter character patterns. Every encounter will be replaced by
a whitespace.
+ *
+ * <p>Four forward cursor passes reproduce, byte for byte, the output of the
former regex
+ * implementation:</p>
+ * <ol>
+ * <li>a {@code #} or {@code @} followed by at least one non-whitespace
character, together
+ * with the whole following non-whitespace run, becomes one space (the
whitespace class is
+ * the six ASCII characters the former {@code \S} complemented);</li>
+ * <li>each maximal sequence of {@code rt} units (case-insensitive, each
followed by a space or
+ * colon) becomes one space when it starts on a word boundary as the JDK
regex engine
+ * defined it for {@code \b};</li>
+ * <li>each emoticon, eyes {@code :}, {@code ;} or {@code x}, an optional
{@code -} nose, and a
+ * mouth out of {@code ( ) d o p} (case-insensitive), becomes one space,
including inside
+ * words;</li>
+ * <li>laughter, a run of {@code h}/{@code j}, a run of vowels, and at least
one repetition of
+ * the two runs' last characters, shrinks to those two characters twice
+ * ({@code "hahaha"} to {@code "haha"}), comparing ASCII
case-insensitively.</li>
+ * </ol>
*/
public class TwitterCharSequenceNormalizer implements CharSequenceNormalizer {
private static final long serialVersionUID = -8155452559337913929L;
-
- private static final Pattern HASH_USER_REGEX =
- Pattern.compile("[#@]\\S+");
-
- private static final Pattern RT_REGEX =
- Pattern.compile("\\b(rt[ :])+", Pattern.CASE_INSENSITIVE);
-
- private static final Pattern FACE_REGEX =
- Pattern.compile("[:;x]-?[()dop]", Pattern.CASE_INSENSITIVE);
-
- private static final Pattern LAUGH_REGEX =
- Pattern.compile("([hj])+([aieou])+(\\1+\\2+)+",
Pattern.CASE_INSENSITIVE);
private static final TwitterCharSequenceNormalizer INSTANCE = new
TwitterCharSequenceNormalizer();
public static TwitterCharSequenceNormalizer getInstance() {
return INSTANCE;
}
+ /**
+ * @throws IllegalArgumentException Thrown if {@code text} is {@code null}.
+ */
@Override
- public CharSequence normalize (CharSequence text) {
- String modified = HASH_USER_REGEX.matcher(text).replaceAll(" ");
- modified = RT_REGEX.matcher(modified).replaceAll(" ");
- modified = FACE_REGEX.matcher(modified).replaceAll(" ");
- modified = LAUGH_REGEX.matcher(modified).replaceAll("$1$2$1$2");
- return modified;
+ public CharSequence normalize(CharSequence text) {
+ if (text == null) {
+ throw new IllegalArgumentException("The text must not be null.");
+ }
+ return
shrinkLaughter(removeEmoticons(removeRetweetMarkers(removeTagsAndHandles(text))));
+ }
+
+ // "[#@]\S+" -> " ": a hash or at sign starts a match only if a
non-whitespace char follows;
+ // the match then swallows the whole non-whitespace run (including further #
and @).
+ private static CharSequence removeTagsAndHandles(CharSequence text) {
+ final int length = text.length();
+ StringBuilder out = null;
+ int i = 0;
+ while (i < length) {
+ final char c = text.charAt(i);
+ if ((c == '#' || c == '@')
+ && i + 1 < length && !AsciiChars.WHITESPACE.contains(text.charAt(i +
1))) {
+ int end = i + 2;
+ while (end < length &&
!AsciiChars.WHITESPACE.contains(text.charAt(end))) {
+ end++;
+ }
+ if (out == null) {
+ out = new StringBuilder(length).append(text, 0, i);
+ }
+ out.append(' ');
+ i = end;
+ } else {
+ if (out != null) {
+ out.append(c);
+ }
+ i++;
+ }
+ }
+ return out == null ? text : out.toString();
+ }
+
+ // "\b(rt[ :])+" (case-insensitive) -> " ": one or more three-char units of
r, t, and a space
+ // or colon, starting where the JDK engine put a word boundary.
+ private static CharSequence removeRetweetMarkers(CharSequence text) {
+ final int length = text.length();
+ StringBuilder out = null;
+ int i = 0;
+ while (i < length) {
+ final char c = text.charAt(i);
+ if ((c == 'r' || c == 'R') && !wordBeforeBlocksBoundary(text, i)) {
+ int end = i;
+ while (end + 3 <= length && isRetweetUnit(text, end)) {
+ end += 3;
+ }
+ if (end > i) {
+ if (out == null) {
+ out = new StringBuilder(length).append(text, 0, i);
+ }
+ out.append(' ');
+ i = end;
+ continue;
+ }
+ }
+ if (out != null) {
+ out.append(c);
+ }
+ i++;
+ }
+ return out == null ? text : out.toString();
+ }
+
+ private static boolean isRetweetUnit(CharSequence text, int at) {
Review Comment:
Done.
##########
opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/TwitterCharSequenceNormalizer.java:
##########
@@ -16,40 +16,263 @@
*/
package opennlp.tools.util.normalizer;
-import java.util.regex.Pattern;
-
/**
- * A {@link TwitterCharSequenceNormalizer} implementation that normalizes text
+ * A {@link CharSequenceNormalizer} implementation that normalizes text
* in terms of Twitter character patterns. Every encounter will be replaced by
a whitespace.
+ *
+ * <p>Four forward cursor passes reproduce, byte for byte, the output of the
former regex
+ * implementation:</p>
+ * <ol>
+ * <li>a {@code #} or {@code @} followed by at least one non-whitespace
character, together
+ * with the whole following non-whitespace run, becomes one space (the
whitespace class is
+ * the six ASCII characters the former {@code \S} complemented);</li>
+ * <li>each maximal sequence of {@code rt} units (case-insensitive, each
followed by a space or
+ * colon) becomes one space when it starts on a word boundary as the JDK
regex engine
+ * defined it for {@code \b};</li>
+ * <li>each emoticon, eyes {@code :}, {@code ;} or {@code x}, an optional
{@code -} nose, and a
+ * mouth out of {@code ( ) d o p} (case-insensitive), becomes one space,
including inside
+ * words;</li>
+ * <li>laughter, a run of {@code h}/{@code j}, a run of vowels, and at least
one repetition of
+ * the two runs' last characters, shrinks to those two characters twice
+ * ({@code "hahaha"} to {@code "haha"}), comparing ASCII
case-insensitively.</li>
+ * </ol>
*/
public class TwitterCharSequenceNormalizer implements CharSequenceNormalizer {
private static final long serialVersionUID = -8155452559337913929L;
-
- private static final Pattern HASH_USER_REGEX =
- Pattern.compile("[#@]\\S+");
-
- private static final Pattern RT_REGEX =
- Pattern.compile("\\b(rt[ :])+", Pattern.CASE_INSENSITIVE);
-
- private static final Pattern FACE_REGEX =
- Pattern.compile("[:;x]-?[()dop]", Pattern.CASE_INSENSITIVE);
-
- private static final Pattern LAUGH_REGEX =
- Pattern.compile("([hj])+([aieou])+(\\1+\\2+)+",
Pattern.CASE_INSENSITIVE);
private static final TwitterCharSequenceNormalizer INSTANCE = new
TwitterCharSequenceNormalizer();
public static TwitterCharSequenceNormalizer getInstance() {
return INSTANCE;
}
+ /**
+ * @throws IllegalArgumentException Thrown if {@code text} is {@code null}.
+ */
@Override
- public CharSequence normalize (CharSequence text) {
- String modified = HASH_USER_REGEX.matcher(text).replaceAll(" ");
- modified = RT_REGEX.matcher(modified).replaceAll(" ");
- modified = FACE_REGEX.matcher(modified).replaceAll(" ");
- modified = LAUGH_REGEX.matcher(modified).replaceAll("$1$2$1$2");
- return modified;
+ public CharSequence normalize(CharSequence text) {
+ if (text == null) {
+ throw new IllegalArgumentException("The text must not be null.");
+ }
+ return
shrinkLaughter(removeEmoticons(removeRetweetMarkers(removeTagsAndHandles(text))));
+ }
+
+ // "[#@]\S+" -> " ": a hash or at sign starts a match only if a
non-whitespace char follows;
+ // the match then swallows the whole non-whitespace run (including further #
and @).
+ private static CharSequence removeTagsAndHandles(CharSequence text) {
+ final int length = text.length();
+ StringBuilder out = null;
+ int i = 0;
+ while (i < length) {
+ final char c = text.charAt(i);
+ if ((c == '#' || c == '@')
+ && i + 1 < length && !AsciiChars.WHITESPACE.contains(text.charAt(i +
1))) {
+ int end = i + 2;
+ while (end < length &&
!AsciiChars.WHITESPACE.contains(text.charAt(end))) {
+ end++;
+ }
+ if (out == null) {
+ out = new StringBuilder(length).append(text, 0, i);
+ }
+ out.append(' ');
+ i = end;
+ } else {
+ if (out != null) {
+ out.append(c);
+ }
+ i++;
+ }
+ }
+ return out == null ? text : out.toString();
+ }
+
+ // "\b(rt[ :])+" (case-insensitive) -> " ": one or more three-char units of
r, t, and a space
+ // or colon, starting where the JDK engine put a word boundary.
+ private static CharSequence removeRetweetMarkers(CharSequence text) {
+ final int length = text.length();
+ StringBuilder out = null;
+ int i = 0;
+ while (i < length) {
+ final char c = text.charAt(i);
+ if ((c == 'r' || c == 'R') && !wordBeforeBlocksBoundary(text, i)) {
+ int end = i;
+ while (end + 3 <= length && isRetweetUnit(text, end)) {
+ end += 3;
+ }
+ if (end > i) {
+ if (out == null) {
+ out = new StringBuilder(length).append(text, 0, i);
+ }
+ out.append(' ');
+ i = end;
+ continue;
+ }
+ }
+ if (out != null) {
+ out.append(c);
+ }
+ i++;
+ }
+ return out == null ? text : out.toString();
+ }
+
+ private static boolean isRetweetUnit(CharSequence text, int at) {
+ final char r = text.charAt(at);
+ final char t = text.charAt(at + 1);
+ final char separator = text.charAt(at + 2);
+ return (r == 'r' || r == 'R') && (t == 't' || t == 'T')
+ && (separator == ' ' || separator == ':');
+ }
+
+ // Mirrors how the JDK regex engine decided whether "\b" held before a word
character at
+ // index (java.util.regex.Pattern.Bound without UNICODE_CHARACTER_CLASS, as
of JDK 21; older
+ // engines treated non-ASCII word characters differently, so the
characterization only holds
+ // on the project's supported runtimes): the boundary is
+ // blocked if the preceding code point is an ASCII word character, or is a
non-spacing mark
+ // with a base character. The base-character scan steps backwards one char
at a time and reads
+ // Character.codePointAt at each position, exactly like the engine, so it
stops on the low
+ // surrogate of a supplementary-plane mark.
+ private static boolean wordBeforeBlocksBoundary(CharSequence text, int
index) {
Review Comment:
Done.
##########
opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/TwitterCharSequenceNormalizer.java:
##########
@@ -16,40 +16,263 @@
*/
package opennlp.tools.util.normalizer;
-import java.util.regex.Pattern;
-
/**
- * A {@link TwitterCharSequenceNormalizer} implementation that normalizes text
+ * A {@link CharSequenceNormalizer} implementation that normalizes text
* in terms of Twitter character patterns. Every encounter will be replaced by
a whitespace.
+ *
+ * <p>Four forward cursor passes reproduce, byte for byte, the output of the
former regex
+ * implementation:</p>
+ * <ol>
+ * <li>a {@code #} or {@code @} followed by at least one non-whitespace
character, together
+ * with the whole following non-whitespace run, becomes one space (the
whitespace class is
+ * the six ASCII characters the former {@code \S} complemented);</li>
+ * <li>each maximal sequence of {@code rt} units (case-insensitive, each
followed by a space or
+ * colon) becomes one space when it starts on a word boundary as the JDK
regex engine
+ * defined it for {@code \b};</li>
+ * <li>each emoticon, eyes {@code :}, {@code ;} or {@code x}, an optional
{@code -} nose, and a
+ * mouth out of {@code ( ) d o p} (case-insensitive), becomes one space,
including inside
+ * words;</li>
+ * <li>laughter, a run of {@code h}/{@code j}, a run of vowels, and at least
one repetition of
+ * the two runs' last characters, shrinks to those two characters twice
+ * ({@code "hahaha"} to {@code "haha"}), comparing ASCII
case-insensitively.</li>
+ * </ol>
*/
public class TwitterCharSequenceNormalizer implements CharSequenceNormalizer {
private static final long serialVersionUID = -8155452559337913929L;
-
- private static final Pattern HASH_USER_REGEX =
- Pattern.compile("[#@]\\S+");
-
- private static final Pattern RT_REGEX =
- Pattern.compile("\\b(rt[ :])+", Pattern.CASE_INSENSITIVE);
-
- private static final Pattern FACE_REGEX =
- Pattern.compile("[:;x]-?[()dop]", Pattern.CASE_INSENSITIVE);
-
- private static final Pattern LAUGH_REGEX =
- Pattern.compile("([hj])+([aieou])+(\\1+\\2+)+",
Pattern.CASE_INSENSITIVE);
private static final TwitterCharSequenceNormalizer INSTANCE = new
TwitterCharSequenceNormalizer();
public static TwitterCharSequenceNormalizer getInstance() {
return INSTANCE;
}
+ /**
+ * @throws IllegalArgumentException Thrown if {@code text} is {@code null}.
+ */
@Override
- public CharSequence normalize (CharSequence text) {
- String modified = HASH_USER_REGEX.matcher(text).replaceAll(" ");
- modified = RT_REGEX.matcher(modified).replaceAll(" ");
- modified = FACE_REGEX.matcher(modified).replaceAll(" ");
- modified = LAUGH_REGEX.matcher(modified).replaceAll("$1$2$1$2");
- return modified;
+ public CharSequence normalize(CharSequence text) {
+ if (text == null) {
+ throw new IllegalArgumentException("The text must not be null.");
+ }
+ return
shrinkLaughter(removeEmoticons(removeRetweetMarkers(removeTagsAndHandles(text))));
+ }
+
+ // "[#@]\S+" -> " ": a hash or at sign starts a match only if a
non-whitespace char follows;
+ // the match then swallows the whole non-whitespace run (including further #
and @).
+ private static CharSequence removeTagsAndHandles(CharSequence text) {
+ final int length = text.length();
+ StringBuilder out = null;
+ int i = 0;
+ while (i < length) {
+ final char c = text.charAt(i);
+ if ((c == '#' || c == '@')
+ && i + 1 < length && !AsciiChars.WHITESPACE.contains(text.charAt(i +
1))) {
+ int end = i + 2;
+ while (end < length &&
!AsciiChars.WHITESPACE.contains(text.charAt(end))) {
+ end++;
+ }
+ if (out == null) {
+ out = new StringBuilder(length).append(text, 0, i);
+ }
+ out.append(' ');
+ i = end;
+ } else {
+ if (out != null) {
+ out.append(c);
+ }
+ i++;
+ }
+ }
+ return out == null ? text : out.toString();
+ }
+
+ // "\b(rt[ :])+" (case-insensitive) -> " ": one or more three-char units of
r, t, and a space
+ // or colon, starting where the JDK engine put a word boundary.
+ private static CharSequence removeRetweetMarkers(CharSequence text) {
+ final int length = text.length();
+ StringBuilder out = null;
+ int i = 0;
+ while (i < length) {
+ final char c = text.charAt(i);
+ if ((c == 'r' || c == 'R') && !wordBeforeBlocksBoundary(text, i)) {
+ int end = i;
+ while (end + 3 <= length && isRetweetUnit(text, end)) {
+ end += 3;
+ }
+ if (end > i) {
+ if (out == null) {
+ out = new StringBuilder(length).append(text, 0, i);
+ }
+ out.append(' ');
+ i = end;
+ continue;
+ }
+ }
+ if (out != null) {
+ out.append(c);
+ }
+ i++;
+ }
+ return out == null ? text : out.toString();
+ }
+
+ private static boolean isRetweetUnit(CharSequence text, int at) {
+ final char r = text.charAt(at);
+ final char t = text.charAt(at + 1);
+ final char separator = text.charAt(at + 2);
+ return (r == 'r' || r == 'R') && (t == 't' || t == 'T')
+ && (separator == ' ' || separator == ':');
+ }
+
+ // Mirrors how the JDK regex engine decided whether "\b" held before a word
character at
+ // index (java.util.regex.Pattern.Bound without UNICODE_CHARACTER_CLASS, as
of JDK 21; older
+ // engines treated non-ASCII word characters differently, so the
characterization only holds
+ // on the project's supported runtimes): the boundary is
+ // blocked if the preceding code point is an ASCII word character, or is a
non-spacing mark
+ // with a base character. The base-character scan steps backwards one char
at a time and reads
+ // Character.codePointAt at each position, exactly like the engine, so it
stops on the low
+ // surrogate of a supplementary-plane mark.
+ private static boolean wordBeforeBlocksBoundary(CharSequence text, int
index) {
+ if (index <= 0) {
+ return false;
+ }
+ final int before = Character.codePointBefore(text, index);
+ if (isAsciiWord(before)) {
+ return true;
+ }
+ if (Character.getType(before) == Character.NON_SPACING_MARK) {
+ for (int x = index - 1; x >= 0; x--) {
+ final int codePoint = Character.codePointAt(text, x);
+ if (Character.isLetterOrDigit(codePoint)) {
+ return true;
+ }
+ if (Character.getType(codePoint) != Character.NON_SPACING_MARK) {
+ return false;
+ }
+ }
+ }
+ return false;
+ }
+
+ private static boolean isAsciiWord(int codePoint) {
Review Comment:
Done.
##########
opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/TwitterCharSequenceNormalizer.java:
##########
@@ -16,40 +16,263 @@
*/
package opennlp.tools.util.normalizer;
-import java.util.regex.Pattern;
-
/**
- * A {@link TwitterCharSequenceNormalizer} implementation that normalizes text
+ * A {@link CharSequenceNormalizer} implementation that normalizes text
* in terms of Twitter character patterns. Every encounter will be replaced by
a whitespace.
+ *
+ * <p>Four forward cursor passes reproduce, byte for byte, the output of the
former regex
+ * implementation:</p>
+ * <ol>
+ * <li>a {@code #} or {@code @} followed by at least one non-whitespace
character, together
+ * with the whole following non-whitespace run, becomes one space (the
whitespace class is
+ * the six ASCII characters the former {@code \S} complemented);</li>
+ * <li>each maximal sequence of {@code rt} units (case-insensitive, each
followed by a space or
+ * colon) becomes one space when it starts on a word boundary as the JDK
regex engine
+ * defined it for {@code \b};</li>
+ * <li>each emoticon, eyes {@code :}, {@code ;} or {@code x}, an optional
{@code -} nose, and a
+ * mouth out of {@code ( ) d o p} (case-insensitive), becomes one space,
including inside
+ * words;</li>
+ * <li>laughter, a run of {@code h}/{@code j}, a run of vowels, and at least
one repetition of
+ * the two runs' last characters, shrinks to those two characters twice
+ * ({@code "hahaha"} to {@code "haha"}), comparing ASCII
case-insensitively.</li>
+ * </ol>
*/
public class TwitterCharSequenceNormalizer implements CharSequenceNormalizer {
private static final long serialVersionUID = -8155452559337913929L;
-
- private static final Pattern HASH_USER_REGEX =
- Pattern.compile("[#@]\\S+");
-
- private static final Pattern RT_REGEX =
- Pattern.compile("\\b(rt[ :])+", Pattern.CASE_INSENSITIVE);
-
- private static final Pattern FACE_REGEX =
- Pattern.compile("[:;x]-?[()dop]", Pattern.CASE_INSENSITIVE);
-
- private static final Pattern LAUGH_REGEX =
- Pattern.compile("([hj])+([aieou])+(\\1+\\2+)+",
Pattern.CASE_INSENSITIVE);
private static final TwitterCharSequenceNormalizer INSTANCE = new
TwitterCharSequenceNormalizer();
public static TwitterCharSequenceNormalizer getInstance() {
return INSTANCE;
}
+ /**
+ * @throws IllegalArgumentException Thrown if {@code text} is {@code null}.
+ */
@Override
- public CharSequence normalize (CharSequence text) {
- String modified = HASH_USER_REGEX.matcher(text).replaceAll(" ");
- modified = RT_REGEX.matcher(modified).replaceAll(" ");
- modified = FACE_REGEX.matcher(modified).replaceAll(" ");
- modified = LAUGH_REGEX.matcher(modified).replaceAll("$1$2$1$2");
- return modified;
+ public CharSequence normalize(CharSequence text) {
+ if (text == null) {
+ throw new IllegalArgumentException("The text must not be null.");
+ }
+ return
shrinkLaughter(removeEmoticons(removeRetweetMarkers(removeTagsAndHandles(text))));
+ }
+
+ // "[#@]\S+" -> " ": a hash or at sign starts a match only if a
non-whitespace char follows;
+ // the match then swallows the whole non-whitespace run (including further #
and @).
+ private static CharSequence removeTagsAndHandles(CharSequence text) {
+ final int length = text.length();
+ StringBuilder out = null;
+ int i = 0;
+ while (i < length) {
+ final char c = text.charAt(i);
+ if ((c == '#' || c == '@')
+ && i + 1 < length && !AsciiChars.WHITESPACE.contains(text.charAt(i +
1))) {
+ int end = i + 2;
+ while (end < length &&
!AsciiChars.WHITESPACE.contains(text.charAt(end))) {
+ end++;
+ }
+ if (out == null) {
+ out = new StringBuilder(length).append(text, 0, i);
+ }
+ out.append(' ');
+ i = end;
+ } else {
+ if (out != null) {
+ out.append(c);
+ }
+ i++;
+ }
+ }
+ return out == null ? text : out.toString();
+ }
+
+ // "\b(rt[ :])+" (case-insensitive) -> " ": one or more three-char units of
r, t, and a space
+ // or colon, starting where the JDK engine put a word boundary.
+ private static CharSequence removeRetweetMarkers(CharSequence text) {
+ final int length = text.length();
+ StringBuilder out = null;
+ int i = 0;
+ while (i < length) {
+ final char c = text.charAt(i);
+ if ((c == 'r' || c == 'R') && !wordBeforeBlocksBoundary(text, i)) {
+ int end = i;
+ while (end + 3 <= length && isRetweetUnit(text, end)) {
+ end += 3;
+ }
+ if (end > i) {
+ if (out == null) {
+ out = new StringBuilder(length).append(text, 0, i);
+ }
+ out.append(' ');
+ i = end;
+ continue;
+ }
+ }
+ if (out != null) {
+ out.append(c);
+ }
+ i++;
+ }
+ return out == null ? text : out.toString();
+ }
+
+ private static boolean isRetweetUnit(CharSequence text, int at) {
+ final char r = text.charAt(at);
+ final char t = text.charAt(at + 1);
+ final char separator = text.charAt(at + 2);
+ return (r == 'r' || r == 'R') && (t == 't' || t == 'T')
+ && (separator == ' ' || separator == ':');
+ }
+
+ // Mirrors how the JDK regex engine decided whether "\b" held before a word
character at
+ // index (java.util.regex.Pattern.Bound without UNICODE_CHARACTER_CLASS, as
of JDK 21; older
+ // engines treated non-ASCII word characters differently, so the
characterization only holds
+ // on the project's supported runtimes): the boundary is
+ // blocked if the preceding code point is an ASCII word character, or is a
non-spacing mark
+ // with a base character. The base-character scan steps backwards one char
at a time and reads
+ // Character.codePointAt at each position, exactly like the engine, so it
stops on the low
+ // surrogate of a supplementary-plane mark.
+ private static boolean wordBeforeBlocksBoundary(CharSequence text, int
index) {
+ if (index <= 0) {
+ return false;
+ }
+ final int before = Character.codePointBefore(text, index);
+ if (isAsciiWord(before)) {
+ return true;
+ }
+ if (Character.getType(before) == Character.NON_SPACING_MARK) {
+ for (int x = index - 1; x >= 0; x--) {
+ final int codePoint = Character.codePointAt(text, x);
+ if (Character.isLetterOrDigit(codePoint)) {
+ return true;
+ }
+ if (Character.getType(codePoint) != Character.NON_SPACING_MARK) {
+ return false;
+ }
+ }
+ }
+ return false;
+ }
+
+ private static boolean isAsciiWord(int codePoint) {
+ return codePoint >= 'a' && codePoint <= 'z'
+ || codePoint >= 'A' && codePoint <= 'Z'
+ || codePoint >= '0' && codePoint <= '9'
+ || codePoint == '_';
+ }
+
+ // "[:;x]-?[()dop]" (case-insensitive) -> " ": eyes, an optional nose, and a
mouth. The
+ // optional nose is tried first, like the greedy "-?"; a bare "-" is never a
mouth.
+ private static CharSequence removeEmoticons(CharSequence text) {
Review Comment:
Done.
##########
opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/TwitterCharSequenceNormalizer.java:
##########
@@ -16,40 +16,263 @@
*/
package opennlp.tools.util.normalizer;
-import java.util.regex.Pattern;
-
/**
- * A {@link TwitterCharSequenceNormalizer} implementation that normalizes text
+ * A {@link CharSequenceNormalizer} implementation that normalizes text
* in terms of Twitter character patterns. Every encounter will be replaced by
a whitespace.
+ *
+ * <p>Four forward cursor passes reproduce, byte for byte, the output of the
former regex
+ * implementation:</p>
+ * <ol>
+ * <li>a {@code #} or {@code @} followed by at least one non-whitespace
character, together
+ * with the whole following non-whitespace run, becomes one space (the
whitespace class is
+ * the six ASCII characters the former {@code \S} complemented);</li>
+ * <li>each maximal sequence of {@code rt} units (case-insensitive, each
followed by a space or
+ * colon) becomes one space when it starts on a word boundary as the JDK
regex engine
+ * defined it for {@code \b};</li>
+ * <li>each emoticon, eyes {@code :}, {@code ;} or {@code x}, an optional
{@code -} nose, and a
+ * mouth out of {@code ( ) d o p} (case-insensitive), becomes one space,
including inside
+ * words;</li>
+ * <li>laughter, a run of {@code h}/{@code j}, a run of vowels, and at least
one repetition of
+ * the two runs' last characters, shrinks to those two characters twice
+ * ({@code "hahaha"} to {@code "haha"}), comparing ASCII
case-insensitively.</li>
+ * </ol>
*/
public class TwitterCharSequenceNormalizer implements CharSequenceNormalizer {
private static final long serialVersionUID = -8155452559337913929L;
-
- private static final Pattern HASH_USER_REGEX =
- Pattern.compile("[#@]\\S+");
-
- private static final Pattern RT_REGEX =
- Pattern.compile("\\b(rt[ :])+", Pattern.CASE_INSENSITIVE);
-
- private static final Pattern FACE_REGEX =
- Pattern.compile("[:;x]-?[()dop]", Pattern.CASE_INSENSITIVE);
-
- private static final Pattern LAUGH_REGEX =
- Pattern.compile("([hj])+([aieou])+(\\1+\\2+)+",
Pattern.CASE_INSENSITIVE);
private static final TwitterCharSequenceNormalizer INSTANCE = new
TwitterCharSequenceNormalizer();
public static TwitterCharSequenceNormalizer getInstance() {
return INSTANCE;
}
+ /**
+ * @throws IllegalArgumentException Thrown if {@code text} is {@code null}.
+ */
@Override
- public CharSequence normalize (CharSequence text) {
- String modified = HASH_USER_REGEX.matcher(text).replaceAll(" ");
- modified = RT_REGEX.matcher(modified).replaceAll(" ");
- modified = FACE_REGEX.matcher(modified).replaceAll(" ");
- modified = LAUGH_REGEX.matcher(modified).replaceAll("$1$2$1$2");
- return modified;
+ public CharSequence normalize(CharSequence text) {
+ if (text == null) {
+ throw new IllegalArgumentException("The text must not be null.");
+ }
+ return
shrinkLaughter(removeEmoticons(removeRetweetMarkers(removeTagsAndHandles(text))));
+ }
+
+ // "[#@]\S+" -> " ": a hash or at sign starts a match only if a
non-whitespace char follows;
+ // the match then swallows the whole non-whitespace run (including further #
and @).
+ private static CharSequence removeTagsAndHandles(CharSequence text) {
+ final int length = text.length();
+ StringBuilder out = null;
+ int i = 0;
+ while (i < length) {
+ final char c = text.charAt(i);
+ if ((c == '#' || c == '@')
+ && i + 1 < length && !AsciiChars.WHITESPACE.contains(text.charAt(i +
1))) {
+ int end = i + 2;
+ while (end < length &&
!AsciiChars.WHITESPACE.contains(text.charAt(end))) {
+ end++;
+ }
+ if (out == null) {
+ out = new StringBuilder(length).append(text, 0, i);
+ }
+ out.append(' ');
+ i = end;
+ } else {
+ if (out != null) {
+ out.append(c);
+ }
+ i++;
+ }
+ }
+ return out == null ? text : out.toString();
+ }
+
+ // "\b(rt[ :])+" (case-insensitive) -> " ": one or more three-char units of
r, t, and a space
+ // or colon, starting where the JDK engine put a word boundary.
+ private static CharSequence removeRetweetMarkers(CharSequence text) {
+ final int length = text.length();
+ StringBuilder out = null;
+ int i = 0;
+ while (i < length) {
+ final char c = text.charAt(i);
+ if ((c == 'r' || c == 'R') && !wordBeforeBlocksBoundary(text, i)) {
+ int end = i;
+ while (end + 3 <= length && isRetweetUnit(text, end)) {
+ end += 3;
+ }
+ if (end > i) {
+ if (out == null) {
+ out = new StringBuilder(length).append(text, 0, i);
+ }
+ out.append(' ');
+ i = end;
+ continue;
+ }
+ }
+ if (out != null) {
+ out.append(c);
+ }
+ i++;
+ }
+ return out == null ? text : out.toString();
+ }
+
+ private static boolean isRetweetUnit(CharSequence text, int at) {
+ final char r = text.charAt(at);
+ final char t = text.charAt(at + 1);
+ final char separator = text.charAt(at + 2);
+ return (r == 'r' || r == 'R') && (t == 't' || t == 'T')
+ && (separator == ' ' || separator == ':');
+ }
+
+ // Mirrors how the JDK regex engine decided whether "\b" held before a word
character at
+ // index (java.util.regex.Pattern.Bound without UNICODE_CHARACTER_CLASS, as
of JDK 21; older
+ // engines treated non-ASCII word characters differently, so the
characterization only holds
+ // on the project's supported runtimes): the boundary is
+ // blocked if the preceding code point is an ASCII word character, or is a
non-spacing mark
+ // with a base character. The base-character scan steps backwards one char
at a time and reads
+ // Character.codePointAt at each position, exactly like the engine, so it
stops on the low
+ // surrogate of a supplementary-plane mark.
+ private static boolean wordBeforeBlocksBoundary(CharSequence text, int
index) {
+ if (index <= 0) {
+ return false;
+ }
+ final int before = Character.codePointBefore(text, index);
+ if (isAsciiWord(before)) {
+ return true;
+ }
+ if (Character.getType(before) == Character.NON_SPACING_MARK) {
+ for (int x = index - 1; x >= 0; x--) {
+ final int codePoint = Character.codePointAt(text, x);
+ if (Character.isLetterOrDigit(codePoint)) {
+ return true;
+ }
+ if (Character.getType(codePoint) != Character.NON_SPACING_MARK) {
+ return false;
+ }
+ }
+ }
+ return false;
+ }
+
+ private static boolean isAsciiWord(int codePoint) {
+ return codePoint >= 'a' && codePoint <= 'z'
+ || codePoint >= 'A' && codePoint <= 'Z'
+ || codePoint >= '0' && codePoint <= '9'
+ || codePoint == '_';
+ }
+
+ // "[:;x]-?[()dop]" (case-insensitive) -> " ": eyes, an optional nose, and a
mouth. The
+ // optional nose is tried first, like the greedy "-?"; a bare "-" is never a
mouth.
+ private static CharSequence removeEmoticons(CharSequence text) {
+ final int length = text.length();
+ StringBuilder out = null;
+ int i = 0;
+ while (i < length) {
+ final char c = text.charAt(i);
+ if (isEmoticonEyes(c)) {
+ if (i + 2 < length && text.charAt(i + 1) == '-' &&
isEmoticonMouth(text.charAt(i + 2))) {
+ if (out == null) {
+ out = new StringBuilder(length).append(text, 0, i);
+ }
+ out.append(' ');
+ i += 3;
+ continue;
+ }
+ if (i + 1 < length && isEmoticonMouth(text.charAt(i + 1))) {
+ if (out == null) {
+ out = new StringBuilder(length).append(text, 0, i);
+ }
+ out.append(' ');
+ i += 2;
+ continue;
+ }
+ }
+ if (out != null) {
+ out.append(c);
+ }
+ i++;
+ }
+ return out == null ? text : out.toString();
+ }
+
+ private static boolean isEmoticonEyes(char c) {
Review Comment:
Done.
##########
opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/TwitterCharSequenceNormalizer.java:
##########
@@ -16,40 +16,263 @@
*/
package opennlp.tools.util.normalizer;
-import java.util.regex.Pattern;
-
/**
- * A {@link TwitterCharSequenceNormalizer} implementation that normalizes text
+ * A {@link CharSequenceNormalizer} implementation that normalizes text
* in terms of Twitter character patterns. Every encounter will be replaced by
a whitespace.
+ *
+ * <p>Four forward cursor passes reproduce, byte for byte, the output of the
former regex
+ * implementation:</p>
+ * <ol>
+ * <li>a {@code #} or {@code @} followed by at least one non-whitespace
character, together
+ * with the whole following non-whitespace run, becomes one space (the
whitespace class is
+ * the six ASCII characters the former {@code \S} complemented);</li>
+ * <li>each maximal sequence of {@code rt} units (case-insensitive, each
followed by a space or
+ * colon) becomes one space when it starts on a word boundary as the JDK
regex engine
+ * defined it for {@code \b};</li>
+ * <li>each emoticon, eyes {@code :}, {@code ;} or {@code x}, an optional
{@code -} nose, and a
+ * mouth out of {@code ( ) d o p} (case-insensitive), becomes one space,
including inside
+ * words;</li>
+ * <li>laughter, a run of {@code h}/{@code j}, a run of vowels, and at least
one repetition of
+ * the two runs' last characters, shrinks to those two characters twice
+ * ({@code "hahaha"} to {@code "haha"}), comparing ASCII
case-insensitively.</li>
+ * </ol>
*/
public class TwitterCharSequenceNormalizer implements CharSequenceNormalizer {
private static final long serialVersionUID = -8155452559337913929L;
-
- private static final Pattern HASH_USER_REGEX =
- Pattern.compile("[#@]\\S+");
-
- private static final Pattern RT_REGEX =
- Pattern.compile("\\b(rt[ :])+", Pattern.CASE_INSENSITIVE);
-
- private static final Pattern FACE_REGEX =
- Pattern.compile("[:;x]-?[()dop]", Pattern.CASE_INSENSITIVE);
-
- private static final Pattern LAUGH_REGEX =
- Pattern.compile("([hj])+([aieou])+(\\1+\\2+)+",
Pattern.CASE_INSENSITIVE);
private static final TwitterCharSequenceNormalizer INSTANCE = new
TwitterCharSequenceNormalizer();
public static TwitterCharSequenceNormalizer getInstance() {
return INSTANCE;
}
+ /**
+ * @throws IllegalArgumentException Thrown if {@code text} is {@code null}.
+ */
@Override
- public CharSequence normalize (CharSequence text) {
- String modified = HASH_USER_REGEX.matcher(text).replaceAll(" ");
- modified = RT_REGEX.matcher(modified).replaceAll(" ");
- modified = FACE_REGEX.matcher(modified).replaceAll(" ");
- modified = LAUGH_REGEX.matcher(modified).replaceAll("$1$2$1$2");
- return modified;
+ public CharSequence normalize(CharSequence text) {
+ if (text == null) {
+ throw new IllegalArgumentException("The text must not be null.");
+ }
+ return
shrinkLaughter(removeEmoticons(removeRetweetMarkers(removeTagsAndHandles(text))));
+ }
+
+ // "[#@]\S+" -> " ": a hash or at sign starts a match only if a
non-whitespace char follows;
+ // the match then swallows the whole non-whitespace run (including further #
and @).
+ private static CharSequence removeTagsAndHandles(CharSequence text) {
+ final int length = text.length();
+ StringBuilder out = null;
+ int i = 0;
+ while (i < length) {
+ final char c = text.charAt(i);
+ if ((c == '#' || c == '@')
+ && i + 1 < length && !AsciiChars.WHITESPACE.contains(text.charAt(i +
1))) {
+ int end = i + 2;
+ while (end < length &&
!AsciiChars.WHITESPACE.contains(text.charAt(end))) {
+ end++;
+ }
+ if (out == null) {
+ out = new StringBuilder(length).append(text, 0, i);
+ }
+ out.append(' ');
+ i = end;
+ } else {
+ if (out != null) {
+ out.append(c);
+ }
+ i++;
+ }
+ }
+ return out == null ? text : out.toString();
+ }
+
+ // "\b(rt[ :])+" (case-insensitive) -> " ": one or more three-char units of
r, t, and a space
+ // or colon, starting where the JDK engine put a word boundary.
+ private static CharSequence removeRetweetMarkers(CharSequence text) {
+ final int length = text.length();
+ StringBuilder out = null;
+ int i = 0;
+ while (i < length) {
+ final char c = text.charAt(i);
+ if ((c == 'r' || c == 'R') && !wordBeforeBlocksBoundary(text, i)) {
+ int end = i;
+ while (end + 3 <= length && isRetweetUnit(text, end)) {
+ end += 3;
+ }
+ if (end > i) {
+ if (out == null) {
+ out = new StringBuilder(length).append(text, 0, i);
+ }
+ out.append(' ');
+ i = end;
+ continue;
+ }
+ }
+ if (out != null) {
+ out.append(c);
+ }
+ i++;
+ }
+ return out == null ? text : out.toString();
+ }
+
+ private static boolean isRetweetUnit(CharSequence text, int at) {
+ final char r = text.charAt(at);
+ final char t = text.charAt(at + 1);
+ final char separator = text.charAt(at + 2);
+ return (r == 'r' || r == 'R') && (t == 't' || t == 'T')
+ && (separator == ' ' || separator == ':');
+ }
+
+ // Mirrors how the JDK regex engine decided whether "\b" held before a word
character at
+ // index (java.util.regex.Pattern.Bound without UNICODE_CHARACTER_CLASS, as
of JDK 21; older
+ // engines treated non-ASCII word characters differently, so the
characterization only holds
+ // on the project's supported runtimes): the boundary is
+ // blocked if the preceding code point is an ASCII word character, or is a
non-spacing mark
+ // with a base character. The base-character scan steps backwards one char
at a time and reads
+ // Character.codePointAt at each position, exactly like the engine, so it
stops on the low
+ // surrogate of a supplementary-plane mark.
+ private static boolean wordBeforeBlocksBoundary(CharSequence text, int
index) {
+ if (index <= 0) {
+ return false;
+ }
+ final int before = Character.codePointBefore(text, index);
+ if (isAsciiWord(before)) {
+ return true;
+ }
+ if (Character.getType(before) == Character.NON_SPACING_MARK) {
+ for (int x = index - 1; x >= 0; x--) {
+ final int codePoint = Character.codePointAt(text, x);
+ if (Character.isLetterOrDigit(codePoint)) {
+ return true;
+ }
+ if (Character.getType(codePoint) != Character.NON_SPACING_MARK) {
+ return false;
+ }
+ }
+ }
+ return false;
+ }
+
+ private static boolean isAsciiWord(int codePoint) {
+ return codePoint >= 'a' && codePoint <= 'z'
+ || codePoint >= 'A' && codePoint <= 'Z'
+ || codePoint >= '0' && codePoint <= '9'
+ || codePoint == '_';
+ }
+
+ // "[:;x]-?[()dop]" (case-insensitive) -> " ": eyes, an optional nose, and a
mouth. The
+ // optional nose is tried first, like the greedy "-?"; a bare "-" is never a
mouth.
+ private static CharSequence removeEmoticons(CharSequence text) {
+ final int length = text.length();
+ StringBuilder out = null;
+ int i = 0;
+ while (i < length) {
+ final char c = text.charAt(i);
+ if (isEmoticonEyes(c)) {
+ if (i + 2 < length && text.charAt(i + 1) == '-' &&
isEmoticonMouth(text.charAt(i + 2))) {
+ if (out == null) {
+ out = new StringBuilder(length).append(text, 0, i);
+ }
+ out.append(' ');
+ i += 3;
+ continue;
+ }
+ if (i + 1 < length && isEmoticonMouth(text.charAt(i + 1))) {
+ if (out == null) {
+ out = new StringBuilder(length).append(text, 0, i);
+ }
+ out.append(' ');
+ i += 2;
+ continue;
+ }
+ }
+ if (out != null) {
+ out.append(c);
+ }
+ i++;
+ }
+ return out == null ? text : out.toString();
+ }
+
+ private static boolean isEmoticonEyes(char c) {
+ return c == ':' || c == ';' || c == 'x' || c == 'X';
+ }
+
+ private static boolean isEmoticonMouth(char c) {
Review Comment:
Done.
##########
opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/TwitterCharSequenceNormalizer.java:
##########
@@ -16,40 +16,263 @@
*/
package opennlp.tools.util.normalizer;
-import java.util.regex.Pattern;
-
/**
- * A {@link TwitterCharSequenceNormalizer} implementation that normalizes text
+ * A {@link CharSequenceNormalizer} implementation that normalizes text
* in terms of Twitter character patterns. Every encounter will be replaced by
a whitespace.
+ *
+ * <p>Four forward cursor passes reproduce, byte for byte, the output of the
former regex
+ * implementation:</p>
+ * <ol>
+ * <li>a {@code #} or {@code @} followed by at least one non-whitespace
character, together
+ * with the whole following non-whitespace run, becomes one space (the
whitespace class is
+ * the six ASCII characters the former {@code \S} complemented);</li>
+ * <li>each maximal sequence of {@code rt} units (case-insensitive, each
followed by a space or
+ * colon) becomes one space when it starts on a word boundary as the JDK
regex engine
+ * defined it for {@code \b};</li>
+ * <li>each emoticon, eyes {@code :}, {@code ;} or {@code x}, an optional
{@code -} nose, and a
+ * mouth out of {@code ( ) d o p} (case-insensitive), becomes one space,
including inside
+ * words;</li>
+ * <li>laughter, a run of {@code h}/{@code j}, a run of vowels, and at least
one repetition of
+ * the two runs' last characters, shrinks to those two characters twice
+ * ({@code "hahaha"} to {@code "haha"}), comparing ASCII
case-insensitively.</li>
+ * </ol>
*/
public class TwitterCharSequenceNormalizer implements CharSequenceNormalizer {
private static final long serialVersionUID = -8155452559337913929L;
-
- private static final Pattern HASH_USER_REGEX =
- Pattern.compile("[#@]\\S+");
-
- private static final Pattern RT_REGEX =
- Pattern.compile("\\b(rt[ :])+", Pattern.CASE_INSENSITIVE);
-
- private static final Pattern FACE_REGEX =
- Pattern.compile("[:;x]-?[()dop]", Pattern.CASE_INSENSITIVE);
-
- private static final Pattern LAUGH_REGEX =
- Pattern.compile("([hj])+([aieou])+(\\1+\\2+)+",
Pattern.CASE_INSENSITIVE);
private static final TwitterCharSequenceNormalizer INSTANCE = new
TwitterCharSequenceNormalizer();
public static TwitterCharSequenceNormalizer getInstance() {
return INSTANCE;
}
+ /**
+ * @throws IllegalArgumentException Thrown if {@code text} is {@code null}.
+ */
@Override
- public CharSequence normalize (CharSequence text) {
- String modified = HASH_USER_REGEX.matcher(text).replaceAll(" ");
- modified = RT_REGEX.matcher(modified).replaceAll(" ");
- modified = FACE_REGEX.matcher(modified).replaceAll(" ");
- modified = LAUGH_REGEX.matcher(modified).replaceAll("$1$2$1$2");
- return modified;
+ public CharSequence normalize(CharSequence text) {
+ if (text == null) {
+ throw new IllegalArgumentException("The text must not be null.");
+ }
+ return
shrinkLaughter(removeEmoticons(removeRetweetMarkers(removeTagsAndHandles(text))));
+ }
+
+ // "[#@]\S+" -> " ": a hash or at sign starts a match only if a
non-whitespace char follows;
+ // the match then swallows the whole non-whitespace run (including further #
and @).
+ private static CharSequence removeTagsAndHandles(CharSequence text) {
+ final int length = text.length();
+ StringBuilder out = null;
+ int i = 0;
+ while (i < length) {
+ final char c = text.charAt(i);
+ if ((c == '#' || c == '@')
+ && i + 1 < length && !AsciiChars.WHITESPACE.contains(text.charAt(i +
1))) {
+ int end = i + 2;
+ while (end < length &&
!AsciiChars.WHITESPACE.contains(text.charAt(end))) {
+ end++;
+ }
+ if (out == null) {
+ out = new StringBuilder(length).append(text, 0, i);
+ }
+ out.append(' ');
+ i = end;
+ } else {
+ if (out != null) {
+ out.append(c);
+ }
+ i++;
+ }
+ }
+ return out == null ? text : out.toString();
+ }
+
+ // "\b(rt[ :])+" (case-insensitive) -> " ": one or more three-char units of
r, t, and a space
+ // or colon, starting where the JDK engine put a word boundary.
+ private static CharSequence removeRetweetMarkers(CharSequence text) {
+ final int length = text.length();
+ StringBuilder out = null;
+ int i = 0;
+ while (i < length) {
+ final char c = text.charAt(i);
+ if ((c == 'r' || c == 'R') && !wordBeforeBlocksBoundary(text, i)) {
+ int end = i;
+ while (end + 3 <= length && isRetweetUnit(text, end)) {
+ end += 3;
+ }
+ if (end > i) {
+ if (out == null) {
+ out = new StringBuilder(length).append(text, 0, i);
+ }
+ out.append(' ');
+ i = end;
+ continue;
+ }
+ }
+ if (out != null) {
+ out.append(c);
+ }
+ i++;
+ }
+ return out == null ? text : out.toString();
+ }
+
+ private static boolean isRetweetUnit(CharSequence text, int at) {
+ final char r = text.charAt(at);
+ final char t = text.charAt(at + 1);
+ final char separator = text.charAt(at + 2);
+ return (r == 'r' || r == 'R') && (t == 't' || t == 'T')
+ && (separator == ' ' || separator == ':');
+ }
+
+ // Mirrors how the JDK regex engine decided whether "\b" held before a word
character at
+ // index (java.util.regex.Pattern.Bound without UNICODE_CHARACTER_CLASS, as
of JDK 21; older
+ // engines treated non-ASCII word characters differently, so the
characterization only holds
+ // on the project's supported runtimes): the boundary is
+ // blocked if the preceding code point is an ASCII word character, or is a
non-spacing mark
+ // with a base character. The base-character scan steps backwards one char
at a time and reads
+ // Character.codePointAt at each position, exactly like the engine, so it
stops on the low
+ // surrogate of a supplementary-plane mark.
+ private static boolean wordBeforeBlocksBoundary(CharSequence text, int
index) {
+ if (index <= 0) {
+ return false;
+ }
+ final int before = Character.codePointBefore(text, index);
+ if (isAsciiWord(before)) {
+ return true;
+ }
+ if (Character.getType(before) == Character.NON_SPACING_MARK) {
+ for (int x = index - 1; x >= 0; x--) {
+ final int codePoint = Character.codePointAt(text, x);
+ if (Character.isLetterOrDigit(codePoint)) {
+ return true;
+ }
+ if (Character.getType(codePoint) != Character.NON_SPACING_MARK) {
+ return false;
+ }
+ }
+ }
+ return false;
+ }
+
+ private static boolean isAsciiWord(int codePoint) {
+ return codePoint >= 'a' && codePoint <= 'z'
+ || codePoint >= 'A' && codePoint <= 'Z'
+ || codePoint >= '0' && codePoint <= '9'
+ || codePoint == '_';
+ }
+
+ // "[:;x]-?[()dop]" (case-insensitive) -> " ": eyes, an optional nose, and a
mouth. The
+ // optional nose is tried first, like the greedy "-?"; a bare "-" is never a
mouth.
+ private static CharSequence removeEmoticons(CharSequence text) {
+ final int length = text.length();
+ StringBuilder out = null;
+ int i = 0;
+ while (i < length) {
+ final char c = text.charAt(i);
+ if (isEmoticonEyes(c)) {
+ if (i + 2 < length && text.charAt(i + 1) == '-' &&
isEmoticonMouth(text.charAt(i + 2))) {
+ if (out == null) {
+ out = new StringBuilder(length).append(text, 0, i);
+ }
+ out.append(' ');
+ i += 3;
+ continue;
+ }
+ if (i + 1 < length && isEmoticonMouth(text.charAt(i + 1))) {
+ if (out == null) {
+ out = new StringBuilder(length).append(text, 0, i);
+ }
+ out.append(' ');
+ i += 2;
+ continue;
+ }
+ }
+ if (out != null) {
+ out.append(c);
+ }
+ i++;
+ }
+ return out == null ? text : out.toString();
+ }
+
+ private static boolean isEmoticonEyes(char c) {
+ return c == ':' || c == ';' || c == 'x' || c == 'X';
+ }
+
+ private static boolean isEmoticonMouth(char c) {
+ return c == '(' || c == ')' || c == 'd' || c == 'D'
+ || c == 'o' || c == 'O' || c == 'p' || c == 'P';
}
+
+ // "([hj])+([aieou])+(\1+\2+)+" (case-insensitive) -> "$1$2$1$2": a run of
h/j, a run of
+ // vowels, and at least one complete repetition of runs of the two captured
characters (the
+ // last consonant and the last vowel, which is what the repeated groups
held). The four
+ // character sets involved are pairwise disjoint, so the greedy runs never
backtrack into
+ // another viable split; a failed attempt resumes at the next char, like the
regex scan.
+ private static CharSequence shrinkLaughter(CharSequence text) {
Review Comment:
Done.
##########
opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/TwitterCharSequenceNormalizer.java:
##########
@@ -16,40 +16,263 @@
*/
package opennlp.tools.util.normalizer;
-import java.util.regex.Pattern;
-
/**
- * A {@link TwitterCharSequenceNormalizer} implementation that normalizes text
+ * A {@link CharSequenceNormalizer} implementation that normalizes text
* in terms of Twitter character patterns. Every encounter will be replaced by
a whitespace.
+ *
+ * <p>Four forward cursor passes reproduce, byte for byte, the output of the
former regex
+ * implementation:</p>
+ * <ol>
+ * <li>a {@code #} or {@code @} followed by at least one non-whitespace
character, together
+ * with the whole following non-whitespace run, becomes one space (the
whitespace class is
+ * the six ASCII characters the former {@code \S} complemented);</li>
+ * <li>each maximal sequence of {@code rt} units (case-insensitive, each
followed by a space or
+ * colon) becomes one space when it starts on a word boundary as the JDK
regex engine
+ * defined it for {@code \b};</li>
+ * <li>each emoticon, eyes {@code :}, {@code ;} or {@code x}, an optional
{@code -} nose, and a
+ * mouth out of {@code ( ) d o p} (case-insensitive), becomes one space,
including inside
+ * words;</li>
+ * <li>laughter, a run of {@code h}/{@code j}, a run of vowels, and at least
one repetition of
+ * the two runs' last characters, shrinks to those two characters twice
+ * ({@code "hahaha"} to {@code "haha"}), comparing ASCII
case-insensitively.</li>
+ * </ol>
*/
public class TwitterCharSequenceNormalizer implements CharSequenceNormalizer {
private static final long serialVersionUID = -8155452559337913929L;
-
- private static final Pattern HASH_USER_REGEX =
- Pattern.compile("[#@]\\S+");
-
- private static final Pattern RT_REGEX =
- Pattern.compile("\\b(rt[ :])+", Pattern.CASE_INSENSITIVE);
-
- private static final Pattern FACE_REGEX =
- Pattern.compile("[:;x]-?[()dop]", Pattern.CASE_INSENSITIVE);
-
- private static final Pattern LAUGH_REGEX =
- Pattern.compile("([hj])+([aieou])+(\\1+\\2+)+",
Pattern.CASE_INSENSITIVE);
private static final TwitterCharSequenceNormalizer INSTANCE = new
TwitterCharSequenceNormalizer();
public static TwitterCharSequenceNormalizer getInstance() {
return INSTANCE;
}
+ /**
+ * @throws IllegalArgumentException Thrown if {@code text} is {@code null}.
+ */
@Override
- public CharSequence normalize (CharSequence text) {
- String modified = HASH_USER_REGEX.matcher(text).replaceAll(" ");
- modified = RT_REGEX.matcher(modified).replaceAll(" ");
- modified = FACE_REGEX.matcher(modified).replaceAll(" ");
- modified = LAUGH_REGEX.matcher(modified).replaceAll("$1$2$1$2");
- return modified;
+ public CharSequence normalize(CharSequence text) {
+ if (text == null) {
+ throw new IllegalArgumentException("The text must not be null.");
+ }
+ return
shrinkLaughter(removeEmoticons(removeRetweetMarkers(removeTagsAndHandles(text))));
+ }
+
+ // "[#@]\S+" -> " ": a hash or at sign starts a match only if a
non-whitespace char follows;
+ // the match then swallows the whole non-whitespace run (including further #
and @).
+ private static CharSequence removeTagsAndHandles(CharSequence text) {
+ final int length = text.length();
+ StringBuilder out = null;
+ int i = 0;
+ while (i < length) {
+ final char c = text.charAt(i);
+ if ((c == '#' || c == '@')
+ && i + 1 < length && !AsciiChars.WHITESPACE.contains(text.charAt(i +
1))) {
+ int end = i + 2;
+ while (end < length &&
!AsciiChars.WHITESPACE.contains(text.charAt(end))) {
+ end++;
+ }
+ if (out == null) {
+ out = new StringBuilder(length).append(text, 0, i);
+ }
+ out.append(' ');
+ i = end;
+ } else {
+ if (out != null) {
+ out.append(c);
+ }
+ i++;
+ }
+ }
+ return out == null ? text : out.toString();
+ }
+
+ // "\b(rt[ :])+" (case-insensitive) -> " ": one or more three-char units of
r, t, and a space
+ // or colon, starting where the JDK engine put a word boundary.
+ private static CharSequence removeRetweetMarkers(CharSequence text) {
+ final int length = text.length();
+ StringBuilder out = null;
+ int i = 0;
+ while (i < length) {
+ final char c = text.charAt(i);
+ if ((c == 'r' || c == 'R') && !wordBeforeBlocksBoundary(text, i)) {
+ int end = i;
+ while (end + 3 <= length && isRetweetUnit(text, end)) {
+ end += 3;
+ }
+ if (end > i) {
+ if (out == null) {
+ out = new StringBuilder(length).append(text, 0, i);
+ }
+ out.append(' ');
+ i = end;
+ continue;
+ }
+ }
+ if (out != null) {
+ out.append(c);
+ }
+ i++;
+ }
+ return out == null ? text : out.toString();
+ }
+
+ private static boolean isRetweetUnit(CharSequence text, int at) {
+ final char r = text.charAt(at);
+ final char t = text.charAt(at + 1);
+ final char separator = text.charAt(at + 2);
+ return (r == 'r' || r == 'R') && (t == 't' || t == 'T')
+ && (separator == ' ' || separator == ':');
+ }
+
+ // Mirrors how the JDK regex engine decided whether "\b" held before a word
character at
+ // index (java.util.regex.Pattern.Bound without UNICODE_CHARACTER_CLASS, as
of JDK 21; older
+ // engines treated non-ASCII word characters differently, so the
characterization only holds
+ // on the project's supported runtimes): the boundary is
+ // blocked if the preceding code point is an ASCII word character, or is a
non-spacing mark
+ // with a base character. The base-character scan steps backwards one char
at a time and reads
+ // Character.codePointAt at each position, exactly like the engine, so it
stops on the low
+ // surrogate of a supplementary-plane mark.
+ private static boolean wordBeforeBlocksBoundary(CharSequence text, int
index) {
+ if (index <= 0) {
+ return false;
+ }
+ final int before = Character.codePointBefore(text, index);
+ if (isAsciiWord(before)) {
+ return true;
+ }
+ if (Character.getType(before) == Character.NON_SPACING_MARK) {
+ for (int x = index - 1; x >= 0; x--) {
+ final int codePoint = Character.codePointAt(text, x);
+ if (Character.isLetterOrDigit(codePoint)) {
+ return true;
+ }
+ if (Character.getType(codePoint) != Character.NON_SPACING_MARK) {
+ return false;
+ }
+ }
+ }
+ return false;
+ }
+
+ private static boolean isAsciiWord(int codePoint) {
+ return codePoint >= 'a' && codePoint <= 'z'
+ || codePoint >= 'A' && codePoint <= 'Z'
+ || codePoint >= '0' && codePoint <= '9'
+ || codePoint == '_';
+ }
+
+ // "[:;x]-?[()dop]" (case-insensitive) -> " ": eyes, an optional nose, and a
mouth. The
+ // optional nose is tried first, like the greedy "-?"; a bare "-" is never a
mouth.
+ private static CharSequence removeEmoticons(CharSequence text) {
+ final int length = text.length();
+ StringBuilder out = null;
+ int i = 0;
+ while (i < length) {
+ final char c = text.charAt(i);
+ if (isEmoticonEyes(c)) {
+ if (i + 2 < length && text.charAt(i + 1) == '-' &&
isEmoticonMouth(text.charAt(i + 2))) {
+ if (out == null) {
+ out = new StringBuilder(length).append(text, 0, i);
+ }
+ out.append(' ');
+ i += 3;
+ continue;
+ }
+ if (i + 1 < length && isEmoticonMouth(text.charAt(i + 1))) {
+ if (out == null) {
+ out = new StringBuilder(length).append(text, 0, i);
+ }
+ out.append(' ');
+ i += 2;
+ continue;
+ }
+ }
+ if (out != null) {
+ out.append(c);
+ }
+ i++;
+ }
+ return out == null ? text : out.toString();
+ }
+
+ private static boolean isEmoticonEyes(char c) {
+ return c == ':' || c == ';' || c == 'x' || c == 'X';
+ }
+
+ private static boolean isEmoticonMouth(char c) {
+ return c == '(' || c == ')' || c == 'd' || c == 'D'
+ || c == 'o' || c == 'O' || c == 'p' || c == 'P';
}
+
+ // "([hj])+([aieou])+(\1+\2+)+" (case-insensitive) -> "$1$2$1$2": a run of
h/j, a run of
+ // vowels, and at least one complete repetition of runs of the two captured
characters (the
+ // last consonant and the last vowel, which is what the repeated groups
held). The four
+ // character sets involved are pairwise disjoint, so the greedy runs never
backtrack into
+ // another viable split; a failed attempt resumes at the next char, like the
regex scan.
+ private static CharSequence shrinkLaughter(CharSequence text) {
+ final int length = text.length();
+ StringBuilder out = null;
+ int i = 0;
+ while (i < length) {
+ final char c = text.charAt(i);
+ if (isLaughConsonant(c)) {
+ int p = i + 1;
+ while (p < length && isLaughConsonant(text.charAt(p))) {
+ p++;
+ }
+ if (p < length && isLaughVowel(text.charAt(p))) {
+ final char consonant = text.charAt(p - 1);
+ int q = p + 1;
+ while (q < length && isLaughVowel(text.charAt(q))) {
+ q++;
+ }
+ final char vowel = text.charAt(q - 1);
+ int end = q;
+ while (true) {
+ int consonantRunEnd = end;
+ while (consonantRunEnd < length
+ &&
AsciiChars.caseInsensitiveEquals(text.charAt(consonantRunEnd), consonant)) {
+ consonantRunEnd++;
+ }
+ if (consonantRunEnd == end) {
+ break;
+ }
+ int vowelRunEnd = consonantRunEnd;
+ while (vowelRunEnd < length
+ && AsciiChars.caseInsensitiveEquals(text.charAt(vowelRunEnd),
vowel)) {
+ vowelRunEnd++;
+ }
+ if (vowelRunEnd == consonantRunEnd) {
+ break;
+ }
+ end = vowelRunEnd;
+ }
+ if (end > q) {
+ if (out == null) {
+ out = new StringBuilder(length).append(text, 0, i);
+ }
+
out.append(consonant).append(vowel).append(consonant).append(vowel);
+ i = end;
+ continue;
+ }
+ }
+ }
+ if (out != null) {
+ out.append(c);
+ }
+ i++;
+ }
+ return out == null ? text : out.toString();
+ }
+
+ private static boolean isLaughConsonant(char c) {
Review Comment:
Done.
##########
opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/TwitterCharSequenceNormalizer.java:
##########
@@ -16,40 +16,263 @@
*/
package opennlp.tools.util.normalizer;
-import java.util.regex.Pattern;
-
/**
- * A {@link TwitterCharSequenceNormalizer} implementation that normalizes text
+ * A {@link CharSequenceNormalizer} implementation that normalizes text
* in terms of Twitter character patterns. Every encounter will be replaced by
a whitespace.
+ *
+ * <p>Four forward cursor passes reproduce, byte for byte, the output of the
former regex
+ * implementation:</p>
+ * <ol>
+ * <li>a {@code #} or {@code @} followed by at least one non-whitespace
character, together
+ * with the whole following non-whitespace run, becomes one space (the
whitespace class is
+ * the six ASCII characters the former {@code \S} complemented);</li>
+ * <li>each maximal sequence of {@code rt} units (case-insensitive, each
followed by a space or
+ * colon) becomes one space when it starts on a word boundary as the JDK
regex engine
+ * defined it for {@code \b};</li>
+ * <li>each emoticon, eyes {@code :}, {@code ;} or {@code x}, an optional
{@code -} nose, and a
+ * mouth out of {@code ( ) d o p} (case-insensitive), becomes one space,
including inside
+ * words;</li>
+ * <li>laughter, a run of {@code h}/{@code j}, a run of vowels, and at least
one repetition of
+ * the two runs' last characters, shrinks to those two characters twice
+ * ({@code "hahaha"} to {@code "haha"}), comparing ASCII
case-insensitively.</li>
+ * </ol>
*/
public class TwitterCharSequenceNormalizer implements CharSequenceNormalizer {
private static final long serialVersionUID = -8155452559337913929L;
-
- private static final Pattern HASH_USER_REGEX =
- Pattern.compile("[#@]\\S+");
-
- private static final Pattern RT_REGEX =
- Pattern.compile("\\b(rt[ :])+", Pattern.CASE_INSENSITIVE);
-
- private static final Pattern FACE_REGEX =
- Pattern.compile("[:;x]-?[()dop]", Pattern.CASE_INSENSITIVE);
-
- private static final Pattern LAUGH_REGEX =
- Pattern.compile("([hj])+([aieou])+(\\1+\\2+)+",
Pattern.CASE_INSENSITIVE);
private static final TwitterCharSequenceNormalizer INSTANCE = new
TwitterCharSequenceNormalizer();
public static TwitterCharSequenceNormalizer getInstance() {
return INSTANCE;
}
+ /**
+ * @throws IllegalArgumentException Thrown if {@code text} is {@code null}.
+ */
@Override
- public CharSequence normalize (CharSequence text) {
- String modified = HASH_USER_REGEX.matcher(text).replaceAll(" ");
- modified = RT_REGEX.matcher(modified).replaceAll(" ");
- modified = FACE_REGEX.matcher(modified).replaceAll(" ");
- modified = LAUGH_REGEX.matcher(modified).replaceAll("$1$2$1$2");
- return modified;
+ public CharSequence normalize(CharSequence text) {
+ if (text == null) {
+ throw new IllegalArgumentException("The text must not be null.");
+ }
+ return
shrinkLaughter(removeEmoticons(removeRetweetMarkers(removeTagsAndHandles(text))));
+ }
+
+ // "[#@]\S+" -> " ": a hash or at sign starts a match only if a
non-whitespace char follows;
+ // the match then swallows the whole non-whitespace run (including further #
and @).
+ private static CharSequence removeTagsAndHandles(CharSequence text) {
+ final int length = text.length();
+ StringBuilder out = null;
+ int i = 0;
+ while (i < length) {
+ final char c = text.charAt(i);
+ if ((c == '#' || c == '@')
+ && i + 1 < length && !AsciiChars.WHITESPACE.contains(text.charAt(i +
1))) {
+ int end = i + 2;
+ while (end < length &&
!AsciiChars.WHITESPACE.contains(text.charAt(end))) {
+ end++;
+ }
+ if (out == null) {
+ out = new StringBuilder(length).append(text, 0, i);
+ }
+ out.append(' ');
+ i = end;
+ } else {
+ if (out != null) {
+ out.append(c);
+ }
+ i++;
+ }
+ }
+ return out == null ? text : out.toString();
+ }
+
+ // "\b(rt[ :])+" (case-insensitive) -> " ": one or more three-char units of
r, t, and a space
+ // or colon, starting where the JDK engine put a word boundary.
+ private static CharSequence removeRetweetMarkers(CharSequence text) {
+ final int length = text.length();
+ StringBuilder out = null;
+ int i = 0;
+ while (i < length) {
+ final char c = text.charAt(i);
+ if ((c == 'r' || c == 'R') && !wordBeforeBlocksBoundary(text, i)) {
+ int end = i;
+ while (end + 3 <= length && isRetweetUnit(text, end)) {
+ end += 3;
+ }
+ if (end > i) {
+ if (out == null) {
+ out = new StringBuilder(length).append(text, 0, i);
+ }
+ out.append(' ');
+ i = end;
+ continue;
+ }
+ }
+ if (out != null) {
+ out.append(c);
+ }
+ i++;
+ }
+ return out == null ? text : out.toString();
+ }
+
+ private static boolean isRetweetUnit(CharSequence text, int at) {
+ final char r = text.charAt(at);
+ final char t = text.charAt(at + 1);
+ final char separator = text.charAt(at + 2);
+ return (r == 'r' || r == 'R') && (t == 't' || t == 'T')
+ && (separator == ' ' || separator == ':');
+ }
+
+ // Mirrors how the JDK regex engine decided whether "\b" held before a word
character at
+ // index (java.util.regex.Pattern.Bound without UNICODE_CHARACTER_CLASS, as
of JDK 21; older
+ // engines treated non-ASCII word characters differently, so the
characterization only holds
+ // on the project's supported runtimes): the boundary is
+ // blocked if the preceding code point is an ASCII word character, or is a
non-spacing mark
+ // with a base character. The base-character scan steps backwards one char
at a time and reads
+ // Character.codePointAt at each position, exactly like the engine, so it
stops on the low
+ // surrogate of a supplementary-plane mark.
+ private static boolean wordBeforeBlocksBoundary(CharSequence text, int
index) {
+ if (index <= 0) {
+ return false;
+ }
+ final int before = Character.codePointBefore(text, index);
+ if (isAsciiWord(before)) {
+ return true;
+ }
+ if (Character.getType(before) == Character.NON_SPACING_MARK) {
+ for (int x = index - 1; x >= 0; x--) {
+ final int codePoint = Character.codePointAt(text, x);
+ if (Character.isLetterOrDigit(codePoint)) {
+ return true;
+ }
+ if (Character.getType(codePoint) != Character.NON_SPACING_MARK) {
+ return false;
+ }
+ }
+ }
+ return false;
+ }
+
+ private static boolean isAsciiWord(int codePoint) {
+ return codePoint >= 'a' && codePoint <= 'z'
+ || codePoint >= 'A' && codePoint <= 'Z'
+ || codePoint >= '0' && codePoint <= '9'
+ || codePoint == '_';
+ }
+
+ // "[:;x]-?[()dop]" (case-insensitive) -> " ": eyes, an optional nose, and a
mouth. The
+ // optional nose is tried first, like the greedy "-?"; a bare "-" is never a
mouth.
+ private static CharSequence removeEmoticons(CharSequence text) {
+ final int length = text.length();
+ StringBuilder out = null;
+ int i = 0;
+ while (i < length) {
+ final char c = text.charAt(i);
+ if (isEmoticonEyes(c)) {
+ if (i + 2 < length && text.charAt(i + 1) == '-' &&
isEmoticonMouth(text.charAt(i + 2))) {
+ if (out == null) {
+ out = new StringBuilder(length).append(text, 0, i);
+ }
+ out.append(' ');
+ i += 3;
+ continue;
+ }
+ if (i + 1 < length && isEmoticonMouth(text.charAt(i + 1))) {
+ if (out == null) {
+ out = new StringBuilder(length).append(text, 0, i);
+ }
+ out.append(' ');
+ i += 2;
+ continue;
+ }
+ }
+ if (out != null) {
+ out.append(c);
+ }
+ i++;
+ }
+ return out == null ? text : out.toString();
+ }
+
+ private static boolean isEmoticonEyes(char c) {
+ return c == ':' || c == ';' || c == 'x' || c == 'X';
+ }
+
+ private static boolean isEmoticonMouth(char c) {
+ return c == '(' || c == ')' || c == 'd' || c == 'D'
+ || c == 'o' || c == 'O' || c == 'p' || c == 'P';
}
+
+ // "([hj])+([aieou])+(\1+\2+)+" (case-insensitive) -> "$1$2$1$2": a run of
h/j, a run of
+ // vowels, and at least one complete repetition of runs of the two captured
characters (the
+ // last consonant and the last vowel, which is what the repeated groups
held). The four
+ // character sets involved are pairwise disjoint, so the greedy runs never
backtrack into
+ // another viable split; a failed attempt resumes at the next char, like the
regex scan.
+ private static CharSequence shrinkLaughter(CharSequence text) {
+ final int length = text.length();
+ StringBuilder out = null;
+ int i = 0;
+ while (i < length) {
+ final char c = text.charAt(i);
+ if (isLaughConsonant(c)) {
+ int p = i + 1;
+ while (p < length && isLaughConsonant(text.charAt(p))) {
+ p++;
+ }
+ if (p < length && isLaughVowel(text.charAt(p))) {
+ final char consonant = text.charAt(p - 1);
+ int q = p + 1;
+ while (q < length && isLaughVowel(text.charAt(q))) {
+ q++;
+ }
+ final char vowel = text.charAt(q - 1);
+ int end = q;
+ while (true) {
+ int consonantRunEnd = end;
+ while (consonantRunEnd < length
+ &&
AsciiChars.caseInsensitiveEquals(text.charAt(consonantRunEnd), consonant)) {
+ consonantRunEnd++;
+ }
+ if (consonantRunEnd == end) {
+ break;
+ }
+ int vowelRunEnd = consonantRunEnd;
+ while (vowelRunEnd < length
+ && AsciiChars.caseInsensitiveEquals(text.charAt(vowelRunEnd),
vowel)) {
+ vowelRunEnd++;
+ }
+ if (vowelRunEnd == consonantRunEnd) {
+ break;
+ }
+ end = vowelRunEnd;
+ }
+ if (end > q) {
+ if (out == null) {
+ out = new StringBuilder(length).append(text, 0, i);
+ }
+
out.append(consonant).append(vowel).append(consonant).append(vowel);
+ i = end;
+ continue;
+ }
+ }
+ }
+ if (out != null) {
+ out.append(c);
+ }
+ i++;
+ }
+ return out == null ? text : out.toString();
+ }
+
+ private static boolean isLaughConsonant(char c) {
+ return c == 'h' || c == 'H' || c == 'j' || c == 'J';
+ }
+
+ private static boolean isLaughVowel(char c) {
Review Comment:
Done.
##########
opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/TwitterCharSequenceNormalizer.java:
##########
@@ -16,40 +16,263 @@
*/
package opennlp.tools.util.normalizer;
-import java.util.regex.Pattern;
-
/**
- * A {@link TwitterCharSequenceNormalizer} implementation that normalizes text
+ * A {@link CharSequenceNormalizer} implementation that normalizes text
* in terms of Twitter character patterns. Every encounter will be replaced by
a whitespace.
+ *
+ * <p>Four forward cursor passes reproduce, byte for byte, the output of the
former regex
+ * implementation:</p>
+ * <ol>
+ * <li>a {@code #} or {@code @} followed by at least one non-whitespace
character, together
+ * with the whole following non-whitespace run, becomes one space (the
whitespace class is
+ * the six ASCII characters the former {@code \S} complemented);</li>
+ * <li>each maximal sequence of {@code rt} units (case-insensitive, each
followed by a space or
+ * colon) becomes one space when it starts on a word boundary as the JDK
regex engine
+ * defined it for {@code \b};</li>
+ * <li>each emoticon, eyes {@code :}, {@code ;} or {@code x}, an optional
{@code -} nose, and a
+ * mouth out of {@code ( ) d o p} (case-insensitive), becomes one space,
including inside
+ * words;</li>
+ * <li>laughter, a run of {@code h}/{@code j}, a run of vowels, and at least
one repetition of
+ * the two runs' last characters, shrinks to those two characters twice
+ * ({@code "hahaha"} to {@code "haha"}), comparing ASCII
case-insensitively.</li>
+ * </ol>
*/
public class TwitterCharSequenceNormalizer implements CharSequenceNormalizer {
Review Comment:
Done: renamed to SocialMediaCharSequenceNormalizer;
TwitterCharSequenceNormalizer stays as a deprecated alias so existing
references keep working, and the langdetect factory chain uses the new name.
##########
opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/TwitterCharSequenceNormalizer.java:
##########
@@ -16,40 +16,263 @@
*/
package opennlp.tools.util.normalizer;
-import java.util.regex.Pattern;
-
/**
- * A {@link TwitterCharSequenceNormalizer} implementation that normalizes text
+ * A {@link CharSequenceNormalizer} implementation that normalizes text
* in terms of Twitter character patterns. Every encounter will be replaced by
a whitespace.
+ *
+ * <p>Four forward cursor passes reproduce, byte for byte, the output of the
former regex
+ * implementation:</p>
+ * <ol>
+ * <li>a {@code #} or {@code @} followed by at least one non-whitespace
character, together
+ * with the whole following non-whitespace run, becomes one space (the
whitespace class is
+ * the six ASCII characters the former {@code \S} complemented);</li>
+ * <li>each maximal sequence of {@code rt} units (case-insensitive, each
followed by a space or
+ * colon) becomes one space when it starts on a word boundary as the JDK
regex engine
+ * defined it for {@code \b};</li>
+ * <li>each emoticon, eyes {@code :}, {@code ;} or {@code x}, an optional
{@code -} nose, and a
+ * mouth out of {@code ( ) d o p} (case-insensitive), becomes one space,
including inside
+ * words;</li>
+ * <li>laughter, a run of {@code h}/{@code j}, a run of vowels, and at least
one repetition of
+ * the two runs' last characters, shrinks to those two characters twice
+ * ({@code "hahaha"} to {@code "haha"}), comparing ASCII
case-insensitively.</li>
+ * </ol>
*/
public class TwitterCharSequenceNormalizer implements CharSequenceNormalizer {
private static final long serialVersionUID = -8155452559337913929L;
Review Comment:
Done, covered by the rename; the new class has its own serialVersionUID.
##########
opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/UrlCharSequenceNormalizer.java:
##########
@@ -16,29 +16,174 @@
*/
package opennlp.tools.util.normalizer;
-import java.util.regex.Pattern;
-
/**
- * A {@link UrlCharSequenceNormalizer} implementation that normalizes text
+ * A {@link CharSequenceNormalizer} implementation that normalizes text
* in terms of URls and email addresses. Every encounter will be replaced by a
whitespace.
+ *
+ * <p>Two forward cursor passes reproduce, byte for byte, the accept/reject
boundary of the
Review Comment:
Done.
##########
opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/UrlCharSequenceNormalizer.java:
##########
@@ -16,29 +16,174 @@
*/
package opennlp.tools.util.normalizer;
-import java.util.regex.Pattern;
-
/**
- * A {@link UrlCharSequenceNormalizer} implementation that normalizes text
+ * A {@link CharSequenceNormalizer} implementation that normalizes text
* in terms of URls and email addresses. Every encounter will be replaced by a
whitespace.
+ *
+ * <p>Two forward cursor passes reproduce, byte for byte, the accept/reject
boundary of the
+ * former regex implementation:</p>
+ * <ol>
+ * <li>URLs: a lowercase {@code http://} or {@code https://} scheme followed
by at least one
+ * character out of the former body class {@code
[-_.?&~;+=/#0-9A-Za-z]}; the match ends at
+ * the first character outside that class, so a colon (port), percent
escape, at sign
+ * (userinfo), or non-ASCII label cuts it short, exactly as before.</li>
+ * <li>Email addresses: a maximal run of the former local-part class {@code
[-+_.0-9A-Za-z]}
+ * whose left neighbor is outside that class (the lookbehind), an {@code
@}, and then a
+ * maximal run of the former domain class {@code [-.0-9A-Za-z]} that
must not start with a
+ * dot and must span at least two chars. The two-char minimum falls out
of the former
+ * {@code [-0-9A-Za-z]+[-.0-9A-Za-z]+} pair: with no dot following, the
first class lent
+ * its last character to the second through backtracking, which needed a
run of two.</li>
+ * </ol>
*/
public class UrlCharSequenceNormalizer implements CharSequenceNormalizer {
Review Comment:
Done: added a differential test running adversarial URL shapes (userinfo,
ports, percent escapes, IPv6 brackets, backslashes, punycode, stray schemes and
separators) through the former patterns and the scan side by side.
##########
opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/UrlCharSequenceNormalizer.java:
##########
@@ -16,29 +16,174 @@
*/
package opennlp.tools.util.normalizer;
-import java.util.regex.Pattern;
-
/**
- * A {@link UrlCharSequenceNormalizer} implementation that normalizes text
+ * A {@link CharSequenceNormalizer} implementation that normalizes text
* in terms of URls and email addresses. Every encounter will be replaced by a
whitespace.
+ *
+ * <p>Two forward cursor passes reproduce, byte for byte, the accept/reject
boundary of the
+ * former regex implementation:</p>
+ * <ol>
+ * <li>URLs: a lowercase {@code http://} or {@code https://} scheme followed
by at least one
+ * character out of the former body class {@code
[-_.?&~;+=/#0-9A-Za-z]}; the match ends at
+ * the first character outside that class, so a colon (port), percent
escape, at sign
+ * (userinfo), or non-ASCII label cuts it short, exactly as before.</li>
+ * <li>Email addresses: a maximal run of the former local-part class {@code
[-+_.0-9A-Za-z]}
+ * whose left neighbor is outside that class (the lookbehind), an {@code
@}, and then a
+ * maximal run of the former domain class {@code [-.0-9A-Za-z]} that
must not start with a
+ * dot and must span at least two chars. The two-char minimum falls out
of the former
+ * {@code [-0-9A-Za-z]+[-.0-9A-Za-z]+} pair: with no dot following, the
first class lent
+ * its last character to the second through backtracking, which needed a
run of two.</li>
+ * </ol>
*/
public class UrlCharSequenceNormalizer implements CharSequenceNormalizer {
private static final long serialVersionUID = 2023145028634552389L;
- private static final Pattern URL_REGEX =
- Pattern.compile("https?://[-_.?&~;+=/#0-9A-Za-z]+");
- private static final Pattern MAIL_REGEX =
-
Pattern.compile("(?<![-+_.0-9A-Za-z])[-+_.0-9A-Za-z]+@[-0-9A-Za-z]+[-.0-9A-Za-z]+");
+
+ private static final CodePointSet ASCII_ALNUM = CodePointSet.ofRange('0',
'9')
+ .union(CodePointSet.ofRange('A', 'Z'))
+ .union(CodePointSet.ofRange('a', 'z'));
+
+ // The former URL body class: [-_.?&~;+=/#0-9A-Za-z]
+ private static final CodePointSet URL_BODY =
+ ASCII_ALNUM.union(CodePointSet.of('-', '_', '.', '?', '&', '~', ';',
'+', '=', '/', '#'));
+
+ // The former mail local-part class (also the lookbehind class):
[-+_.0-9A-Za-z]
+ private static final CodePointSet MAIL_LOCAL =
+ ASCII_ALNUM.union(CodePointSet.of('-', '+', '_', '.'));
+
+ // The former first domain class: [-0-9A-Za-z]
+ private static final CodePointSet MAIL_DOMAIN_START =
ASCII_ALNUM.union(CodePointSet.of('-'));
+
+ // The former second domain class: [-.0-9A-Za-z]
+ private static final CodePointSet MAIL_DOMAIN =
MAIL_DOMAIN_START.union(CodePointSet.of('.'));
private static final UrlCharSequenceNormalizer INSTANCE = new
UrlCharSequenceNormalizer();
public static UrlCharSequenceNormalizer getInstance() {
return INSTANCE;
}
+ /**
+ * @throws IllegalArgumentException Thrown if {@code text} is {@code null}.
+ */
@Override
- public CharSequence normalize (CharSequence text) {
- String modified = URL_REGEX.matcher(text).replaceAll(" ");
- return MAIL_REGEX.matcher(modified).replaceAll(" ");
+ public CharSequence normalize(CharSequence text) {
+ if (text == null) {
+ throw new IllegalArgumentException("The text must not be null.");
+ }
+ return removeMailAddresses(removeUrls(text));
+ }
+
+ // "https?://[-_.?&~;+=/#0-9A-Za-z]+" -> " "
+ private static CharSequence removeUrls(CharSequence text) {
Review Comment:
Done.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
