krickert commented on code in PR #1151: URL: https://github.com/apache/opennlp/pull/1151#discussion_r3564893152
########## opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/AsciiChars.java: ########## @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.tools.util.normalizer; + +/** + * ASCII character helpers shared by the cursor-scan rewrites of the legacy normalizers, so the + * definitions the former regexes agreed on cannot drift apart between classes. + */ +final class AsciiChars { Review Comment: Done. ########## opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/AsciiChars.java: ########## @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.tools.util.normalizer; + +/** + * ASCII character helpers shared by the cursor-scan rewrites of the legacy normalizers, so the + * definitions the former regexes agreed on cannot drift apart between classes. + */ +final class AsciiChars { + + /** The six characters the former regex {@code \s} class matched. */ Review Comment: Done, listed by name and code point. ########## opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/NumberCharSequenceNormalizer.java: ########## @@ -16,26 +16,35 @@ */ package opennlp.tools.util.normalizer; -import java.util.regex.Pattern; - /** - * A {@link NumberCharSequenceNormalizer} implementation that normalizes text - * in terms of numbers. Every encounter will be replaced by a whitespace. + * A {@link CharSequenceNormalizer} implementation that normalizes text + * in terms of numbers: every maximal run of ASCII digits ({@code 0} to {@code 9}) is replaced + * by a single whitespace. + * + * <p>This reproduces, byte for byte, the output of the former regex implementation Review Comment: Done. ########## opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/NumberCharSequenceNormalizer.java: ########## @@ -16,26 +16,35 @@ */ package opennlp.tools.util.normalizer; -import java.util.regex.Pattern; - /** - * A {@link NumberCharSequenceNormalizer} implementation that normalizes text - * in terms of numbers. Every encounter will be replaced by a whitespace. + * A {@link CharSequenceNormalizer} implementation that normalizes text + * in terms of numbers: every maximal run of ASCII digits ({@code 0} to {@code 9}) is replaced Review Comment: Done: "a1234b56" becomes "a b ", pinned by a test. ########## opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/NumberCharSequenceNormalizer.java: ########## @@ -16,26 +16,35 @@ */ package opennlp.tools.util.normalizer; -import java.util.regex.Pattern; - /** - * A {@link NumberCharSequenceNormalizer} implementation that normalizes text - * in terms of numbers. Every encounter will be replaced by a whitespace. + * A {@link CharSequenceNormalizer} implementation that normalizes text + * in terms of numbers: every maximal run of ASCII digits ({@code 0} to {@code 9}) is replaced + * by a single whitespace. + * + * <p>This reproduces, byte for byte, the output of the former regex implementation + * ({@code "\\d+"} replaced by a space; the {@code \d} class matches ASCII digits only), but runs + * as a single forward cursor scan on the {@link CharClass} engine instead of a regular + * expression. Non-ASCII digits, for example Arabic-Indic or fullwidth digits, are left + * unchanged, exactly as before.</p> */ public class NumberCharSequenceNormalizer implements CharSequenceNormalizer { private static final long serialVersionUID = -782056416383201122L; - - private static final Pattern NUMBER_REGEX = Pattern.compile("\\d+"); + + private static final CharClass ASCII_DIGITS = + CharClass.of(CodePointSet.ofRange('0', '9'), ' '); private static final NumberCharSequenceNormalizer INSTANCE = new NumberCharSequenceNormalizer(); public static NumberCharSequenceNormalizer getInstance() { return INSTANCE; } + /** + * @throws IllegalArgumentException Thrown if {@code text} is {@code null}. Review Comment: Done. ########## opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/ShrinkCharSequenceNormalizer.java: ########## @@ -16,28 +16,131 @@ */ package opennlp.tools.util.normalizer; -import java.util.regex.Pattern; - /** - * A {@link ShrinkCharSequenceNormalizer} implementation that shrinks repeated spaces / chars in text. + * A {@link CharSequenceNormalizer} implementation that shrinks repeated spaces / chars + * in text. + * + * <p>Two forward cursor passes reproduce, byte for byte, the output of the former regex Review Comment: Done, the javadoc describes the behavior in self-contained terms now; the old patterns live on only in the characterization tests. ########## opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/ShrinkCharSequenceNormalizer.java: ########## @@ -16,28 +16,131 @@ */ package opennlp.tools.util.normalizer; -import java.util.regex.Pattern; - /** - * A {@link ShrinkCharSequenceNormalizer} implementation that shrinks repeated spaces / chars in text. + * A {@link CharSequenceNormalizer} implementation that shrinks repeated spaces / chars + * in text. + * + * <p>Two forward cursor passes reproduce, byte for byte, the output of the former regex + * implementation ({@code "\\s{2,}"} replaced by one space, then {@code "(.)\\1{2,}"} with + * {@code CASE_INSENSITIVE} replaced by {@code "$1$1"}, then {@link String#trim()}):</p> + * <ol> + * <li>Each run of two or more ASCII whitespace characters (the six characters the regex + * {@code \s} class matches; this legacy rung predates the Unicode {@code White_Space} + * based {@link CharClass#whitespace()}) collapses to a single space. A lone whitespace + * character is kept as it is.</li> + * <li>Each run of three or more repeats of one code point shrinks to two copies of its first + * occurrence. Repeats compare case-insensitively over ASCII only, and a run never starts + * on a regex line terminator, because the former {@code .} did not match one.</li> + * </ol> + * <p>The result is finally trimmed with {@link String#trim()}, preserving the legacy contract Review Comment: Done. ########## opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/ShrinkCharSequenceNormalizer.java: ########## @@ -16,28 +16,131 @@ */ package opennlp.tools.util.normalizer; -import java.util.regex.Pattern; - /** - * A {@link ShrinkCharSequenceNormalizer} implementation that shrinks repeated spaces / chars in text. + * A {@link CharSequenceNormalizer} implementation that shrinks repeated spaces / chars + * in text. + * + * <p>Two forward cursor passes reproduce, byte for byte, the output of the former regex + * implementation ({@code "\\s{2,}"} replaced by one space, then {@code "(.)\\1{2,}"} with + * {@code CASE_INSENSITIVE} replaced by {@code "$1$1"}, then {@link String#trim()}):</p> + * <ol> + * <li>Each run of two or more ASCII whitespace characters (the six characters the regex + * {@code \s} class matches; this legacy rung predates the Unicode {@code White_Space} + * based {@link CharClass#whitespace()}) collapses to a single space. A lone whitespace + * character is kept as it is.</li> + * <li>Each run of three or more repeats of one code point shrinks to two copies of its first + * occurrence. Repeats compare case-insensitively over ASCII only, and a run never starts + * on a regex line terminator, because the former {@code .} did not match one.</li> + * </ol> + * <p>The result is finally trimmed with {@link String#trim()}, preserving the legacy contract + * that every leading or trailing char up to {@code U+0020} is dropped.</p> */ public class ShrinkCharSequenceNormalizer implements CharSequenceNormalizer { private static final long serialVersionUID = -4511969661556543048L; Review Comment: Done, on all four rewritten classes. ########## opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/ShrinkCharSequenceNormalizer.java: ########## @@ -16,28 +16,131 @@ */ package opennlp.tools.util.normalizer; -import java.util.regex.Pattern; - /** - * A {@link ShrinkCharSequenceNormalizer} implementation that shrinks repeated spaces / chars in text. + * A {@link CharSequenceNormalizer} implementation that shrinks repeated spaces / chars + * in text. + * + * <p>Two forward cursor passes reproduce, byte for byte, the output of the former regex + * implementation ({@code "\\s{2,}"} replaced by one space, then {@code "(.)\\1{2,}"} with + * {@code CASE_INSENSITIVE} replaced by {@code "$1$1"}, then {@link String#trim()}):</p> + * <ol> + * <li>Each run of two or more ASCII whitespace characters (the six characters the regex + * {@code \s} class matches; this legacy rung predates the Unicode {@code White_Space} + * based {@link CharClass#whitespace()}) collapses to a single space. A lone whitespace + * character is kept as it is.</li> + * <li>Each run of three or more repeats of one code point shrinks to two copies of its first + * occurrence. Repeats compare case-insensitively over ASCII only, and a run never starts + * on a regex line terminator, because the former {@code .} did not match one.</li> + * </ol> + * <p>The result is finally trimmed with {@link String#trim()}, preserving the legacy contract + * that every leading or trailing char up to {@code U+0020} is dropped.</p> */ public class ShrinkCharSequenceNormalizer implements CharSequenceNormalizer { private static final long serialVersionUID = -4511969661556543048L; - private static final Pattern REPEATED_CHAR_REGEX = Pattern.compile("(.)\\1{2,}", - Pattern.CASE_INSENSITIVE); - private static final Pattern SPACE_REGEX = Pattern.compile("\\s{2,}", - Pattern.CASE_INSENSITIVE); + // The code points the former "." (without DOTALL) refused to match: the regex line terminators. Review Comment: Done. ########## opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/ShrinkCharSequenceNormalizer.java: ########## @@ -16,28 +16,131 @@ */ package opennlp.tools.util.normalizer; -import java.util.regex.Pattern; - /** - * A {@link ShrinkCharSequenceNormalizer} implementation that shrinks repeated spaces / chars in text. + * A {@link CharSequenceNormalizer} implementation that shrinks repeated spaces / chars + * in text. + * + * <p>Two forward cursor passes reproduce, byte for byte, the output of the former regex + * implementation ({@code "\\s{2,}"} replaced by one space, then {@code "(.)\\1{2,}"} with + * {@code CASE_INSENSITIVE} replaced by {@code "$1$1"}, then {@link String#trim()}):</p> + * <ol> + * <li>Each run of two or more ASCII whitespace characters (the six characters the regex + * {@code \s} class matches; this legacy rung predates the Unicode {@code White_Space} + * based {@link CharClass#whitespace()}) collapses to a single space. A lone whitespace + * character is kept as it is.</li> + * <li>Each run of three or more repeats of one code point shrinks to two copies of its first + * occurrence. Repeats compare case-insensitively over ASCII only, and a run never starts + * on a regex line terminator, because the former {@code .} did not match one.</li> + * </ol> + * <p>The result is finally trimmed with {@link String#trim()}, preserving the legacy contract + * that every leading or trailing char up to {@code U+0020} is dropped.</p> */ public class ShrinkCharSequenceNormalizer implements CharSequenceNormalizer { private static final long serialVersionUID = -4511969661556543048L; - private static final Pattern REPEATED_CHAR_REGEX = Pattern.compile("(.)\\1{2,}", - Pattern.CASE_INSENSITIVE); - private static final Pattern SPACE_REGEX = Pattern.compile("\\s{2,}", - Pattern.CASE_INSENSITIVE); + // The code points the former "." (without DOTALL) refused to match: the regex line terminators. + private static final CodePointSet REGEX_LINE_TERMINATORS = + CodePointSet.of(0x000A, 0x000D, 0x0085, 0x2028, 0x2029); private static final ShrinkCharSequenceNormalizer INSTANCE = new ShrinkCharSequenceNormalizer(); public static ShrinkCharSequenceNormalizer getInstance() { return INSTANCE; } - public CharSequence normalize (CharSequence text) { - text = SPACE_REGEX.matcher(text).replaceAll(" "); - return REPEATED_CHAR_REGEX.matcher(text).replaceAll("$1$1").trim(); + /** + * @throws IllegalArgumentException Thrown if {@code text} is {@code null}. Review Comment: Done. ########## opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/ShrinkCharSequenceNormalizer.java: ########## @@ -16,28 +16,131 @@ */ package opennlp.tools.util.normalizer; -import java.util.regex.Pattern; - /** - * A {@link ShrinkCharSequenceNormalizer} implementation that shrinks repeated spaces / chars in text. + * A {@link CharSequenceNormalizer} implementation that shrinks repeated spaces / chars + * in text. + * + * <p>Two forward cursor passes reproduce, byte for byte, the output of the former regex + * implementation ({@code "\\s{2,}"} replaced by one space, then {@code "(.)\\1{2,}"} with + * {@code CASE_INSENSITIVE} replaced by {@code "$1$1"}, then {@link String#trim()}):</p> + * <ol> + * <li>Each run of two or more ASCII whitespace characters (the six characters the regex + * {@code \s} class matches; this legacy rung predates the Unicode {@code White_Space} + * based {@link CharClass#whitespace()}) collapses to a single space. A lone whitespace + * character is kept as it is.</li> + * <li>Each run of three or more repeats of one code point shrinks to two copies of its first + * occurrence. Repeats compare case-insensitively over ASCII only, and a run never starts + * on a regex line terminator, because the former {@code .} did not match one.</li> + * </ol> + * <p>The result is finally trimmed with {@link String#trim()}, preserving the legacy contract + * that every leading or trailing char up to {@code U+0020} is dropped.</p> */ public class ShrinkCharSequenceNormalizer implements CharSequenceNormalizer { private static final long serialVersionUID = -4511969661556543048L; - private static final Pattern REPEATED_CHAR_REGEX = Pattern.compile("(.)\\1{2,}", - Pattern.CASE_INSENSITIVE); - private static final Pattern SPACE_REGEX = Pattern.compile("\\s{2,}", - Pattern.CASE_INSENSITIVE); + // The code points the former "." (without DOTALL) refused to match: the regex line terminators. + private static final CodePointSet REGEX_LINE_TERMINATORS = + CodePointSet.of(0x000A, 0x000D, 0x0085, 0x2028, 0x2029); private static final ShrinkCharSequenceNormalizer INSTANCE = new ShrinkCharSequenceNormalizer(); public static ShrinkCharSequenceNormalizer getInstance() { return INSTANCE; } - public CharSequence normalize (CharSequence text) { - text = SPACE_REGEX.matcher(text).replaceAll(" "); - return REPEATED_CHAR_REGEX.matcher(text).replaceAll("$1$1").trim(); + /** + * @throws IllegalArgumentException Thrown if {@code text} is {@code null}. + */ + @Override + public CharSequence normalize(CharSequence text) { + if (text == null) { + throw new IllegalArgumentException("The text must not be null."); + } + final CharSequence shrunk = shrinkRepeatedCodePoints(shrinkWhitespace(text)); + // Trim exactly like String.trim did (chars at or below U+0020), without copying when Review Comment: Done, the javadoc states the rule itself now. ########## opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/ShrinkCharSequenceNormalizer.java: ########## @@ -16,28 +16,131 @@ */ package opennlp.tools.util.normalizer; -import java.util.regex.Pattern; - /** - * A {@link ShrinkCharSequenceNormalizer} implementation that shrinks repeated spaces / chars in text. + * A {@link CharSequenceNormalizer} implementation that shrinks repeated spaces / chars + * in text. + * + * <p>Two forward cursor passes reproduce, byte for byte, the output of the former regex + * implementation ({@code "\\s{2,}"} replaced by one space, then {@code "(.)\\1{2,}"} with + * {@code CASE_INSENSITIVE} replaced by {@code "$1$1"}, then {@link String#trim()}):</p> + * <ol> + * <li>Each run of two or more ASCII whitespace characters (the six characters the regex + * {@code \s} class matches; this legacy rung predates the Unicode {@code White_Space} + * based {@link CharClass#whitespace()}) collapses to a single space. A lone whitespace + * character is kept as it is.</li> + * <li>Each run of three or more repeats of one code point shrinks to two copies of its first + * occurrence. Repeats compare case-insensitively over ASCII only, and a run never starts + * on a regex line terminator, because the former {@code .} did not match one.</li> + * </ol> + * <p>The result is finally trimmed with {@link String#trim()}, preserving the legacy contract + * that every leading or trailing char up to {@code U+0020} is dropped.</p> */ public class ShrinkCharSequenceNormalizer implements CharSequenceNormalizer { private static final long serialVersionUID = -4511969661556543048L; - private static final Pattern REPEATED_CHAR_REGEX = Pattern.compile("(.)\\1{2,}", - Pattern.CASE_INSENSITIVE); - private static final Pattern SPACE_REGEX = Pattern.compile("\\s{2,}", - Pattern.CASE_INSENSITIVE); + // The code points the former "." (without DOTALL) refused to match: the regex line terminators. + private static final CodePointSet REGEX_LINE_TERMINATORS = + CodePointSet.of(0x000A, 0x000D, 0x0085, 0x2028, 0x2029); private static final ShrinkCharSequenceNormalizer INSTANCE = new ShrinkCharSequenceNormalizer(); public static ShrinkCharSequenceNormalizer getInstance() { return INSTANCE; } - public CharSequence normalize (CharSequence text) { - text = SPACE_REGEX.matcher(text).replaceAll(" "); - return REPEATED_CHAR_REGEX.matcher(text).replaceAll("$1$1").trim(); + /** + * @throws IllegalArgumentException Thrown if {@code text} is {@code null}. + */ + @Override + public CharSequence normalize(CharSequence text) { + if (text == null) { + throw new IllegalArgumentException("The text must not be null."); + } + final CharSequence shrunk = shrinkRepeatedCodePoints(shrinkWhitespace(text)); + // Trim exactly like String.trim did (chars at or below U+0020), without copying when + // nothing changed anywhere in the pipeline. + int start = 0; + int end = shrunk.length(); + while (start < end && shrunk.charAt(start) <= ' ') { + start++; + } + while (end > start && shrunk.charAt(end - 1) <= ' ') { + end--; + } + if (start == 0 && end == shrunk.length()) { + return shrunk; + } + return shrunk.subSequence(start, end).toString(); + } + + // Replaces each run of two or more ASCII whitespace characters with a single space; a lone + // whitespace character stays as it is (the former "\s{2,}" never matched a run of one). + private static CharSequence shrinkWhitespace(CharSequence text) { Review Comment: Done. ########## opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/ShrinkCharSequenceNormalizer.java: ########## @@ -16,28 +16,131 @@ */ package opennlp.tools.util.normalizer; -import java.util.regex.Pattern; - /** - * A {@link ShrinkCharSequenceNormalizer} implementation that shrinks repeated spaces / chars in text. + * A {@link CharSequenceNormalizer} implementation that shrinks repeated spaces / chars + * in text. + * + * <p>Two forward cursor passes reproduce, byte for byte, the output of the former regex + * implementation ({@code "\\s{2,}"} replaced by one space, then {@code "(.)\\1{2,}"} with + * {@code CASE_INSENSITIVE} replaced by {@code "$1$1"}, then {@link String#trim()}):</p> + * <ol> + * <li>Each run of two or more ASCII whitespace characters (the six characters the regex + * {@code \s} class matches; this legacy rung predates the Unicode {@code White_Space} + * based {@link CharClass#whitespace()}) collapses to a single space. A lone whitespace + * character is kept as it is.</li> + * <li>Each run of three or more repeats of one code point shrinks to two copies of its first + * occurrence. Repeats compare case-insensitively over ASCII only, and a run never starts + * on a regex line terminator, because the former {@code .} did not match one.</li> + * </ol> + * <p>The result is finally trimmed with {@link String#trim()}, preserving the legacy contract + * that every leading or trailing char up to {@code U+0020} is dropped.</p> */ public class ShrinkCharSequenceNormalizer implements CharSequenceNormalizer { private static final long serialVersionUID = -4511969661556543048L; - private static final Pattern REPEATED_CHAR_REGEX = Pattern.compile("(.)\\1{2,}", - Pattern.CASE_INSENSITIVE); - private static final Pattern SPACE_REGEX = Pattern.compile("\\s{2,}", - Pattern.CASE_INSENSITIVE); + // The code points the former "." (without DOTALL) refused to match: the regex line terminators. + private static final CodePointSet REGEX_LINE_TERMINATORS = + CodePointSet.of(0x000A, 0x000D, 0x0085, 0x2028, 0x2029); private static final ShrinkCharSequenceNormalizer INSTANCE = new ShrinkCharSequenceNormalizer(); public static ShrinkCharSequenceNormalizer getInstance() { return INSTANCE; } - public CharSequence normalize (CharSequence text) { - text = SPACE_REGEX.matcher(text).replaceAll(" "); - return REPEATED_CHAR_REGEX.matcher(text).replaceAll("$1$1").trim(); + /** + * @throws IllegalArgumentException Thrown if {@code text} is {@code null}. + */ + @Override + public CharSequence normalize(CharSequence text) { + if (text == null) { + throw new IllegalArgumentException("The text must not be null."); + } + final CharSequence shrunk = shrinkRepeatedCodePoints(shrinkWhitespace(text)); + // Trim exactly like String.trim did (chars at or below U+0020), without copying when + // nothing changed anywhere in the pipeline. + int start = 0; + int end = shrunk.length(); + while (start < end && shrunk.charAt(start) <= ' ') { + start++; + } + while (end > start && shrunk.charAt(end - 1) <= ' ') { + end--; + } + if (start == 0 && end == shrunk.length()) { + return shrunk; + } + return shrunk.subSequence(start, end).toString(); + } + + // Replaces each run of two or more ASCII whitespace characters with a single space; a lone + // whitespace character stays as it is (the former "\s{2,}" never matched a run of one). + private static CharSequence shrinkWhitespace(CharSequence text) { + final int length = text.length(); + StringBuilder out = null; + int i = 0; + while (i < length) { + final int codePoint = Character.codePointAt(text, i); + if (AsciiChars.WHITESPACE.contains(codePoint)) { + int runEnd = i + 1; // members are single-char ASCII + while (runEnd < length && AsciiChars.WHITESPACE.contains(text.charAt(runEnd))) { + runEnd++; + } + if (runEnd - i >= 2) { + if (out == null) { + out = new StringBuilder(length).append(text, 0, i); + } + out.append(' '); + } else if (out != null) { + out.append(text.charAt(i)); + } + i = runEnd; + } else { + if (out != null) { + out.appendCodePoint(codePoint); + } + i += Character.charCount(codePoint); + } + } + return out == null ? text : out.toString(); } + + // Shrinks each run of three or more repeats of one code point to two copies of its first + // occurrence, exactly as the former "(.)\1{2,}" -> "$1$1" pass did: the run's first code point + // must not be a regex line terminator, every repeat advances by the first occurrence's char + // count while comparing whole code points ASCII case-insensitively (mirroring the engine's + // backreference matching), and a failed attempt resumes at the next char, like the regex scan. + private static CharSequence shrinkRepeatedCodePoints(CharSequence text) { Review Comment: Done. ########## opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/TwitterCharSequenceNormalizer.java: ########## @@ -16,40 +16,263 @@ */ package opennlp.tools.util.normalizer; -import java.util.regex.Pattern; - /** - * A {@link TwitterCharSequenceNormalizer} implementation that normalizes text + * A {@link CharSequenceNormalizer} implementation that normalizes text * in terms of Twitter character patterns. Every encounter will be replaced by a whitespace. + * + * <p>Four forward cursor passes reproduce, byte for byte, the output of the former regex Review Comment: Done, same treatment. ########## opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/TwitterCharSequenceNormalizer.java: ########## @@ -16,40 +16,263 @@ */ package opennlp.tools.util.normalizer; -import java.util.regex.Pattern; - /** - * A {@link TwitterCharSequenceNormalizer} implementation that normalizes text + * A {@link CharSequenceNormalizer} implementation that normalizes text * in terms of Twitter character patterns. Every encounter will be replaced by a whitespace. + * + * <p>Four forward cursor passes reproduce, byte for byte, the output of the former regex + * implementation:</p> + * <ol> + * <li>a {@code #} or {@code @} followed by at least one non-whitespace character, together + * with the whole following non-whitespace run, becomes one space (the whitespace class is + * the six ASCII characters the former {@code \S} complemented);</li> + * <li>each maximal sequence of {@code rt} units (case-insensitive, each followed by a space or + * colon) becomes one space when it starts on a word boundary as the JDK regex engine + * defined it for {@code \b};</li> + * <li>each emoticon, eyes {@code :}, {@code ;} or {@code x}, an optional {@code -} nose, and a + * mouth out of {@code ( ) d o p} (case-insensitive), becomes one space, including inside + * words;</li> + * <li>laughter, a run of {@code h}/{@code j}, a run of vowels, and at least one repetition of + * the two runs' last characters, shrinks to those two characters twice + * ({@code "hahaha"} to {@code "haha"}), comparing ASCII case-insensitively.</li> + * </ol> */ public class TwitterCharSequenceNormalizer implements CharSequenceNormalizer { private static final long serialVersionUID = -8155452559337913929L; - - private static final Pattern HASH_USER_REGEX = - Pattern.compile("[#@]\\S+"); - - private static final Pattern RT_REGEX = - Pattern.compile("\\b(rt[ :])+", Pattern.CASE_INSENSITIVE); - - private static final Pattern FACE_REGEX = - Pattern.compile("[:;x]-?[()dop]", Pattern.CASE_INSENSITIVE); - - private static final Pattern LAUGH_REGEX = - Pattern.compile("([hj])+([aieou])+(\\1+\\2+)+", Pattern.CASE_INSENSITIVE); private static final TwitterCharSequenceNormalizer INSTANCE = new TwitterCharSequenceNormalizer(); public static TwitterCharSequenceNormalizer getInstance() { return INSTANCE; } + /** + * @throws IllegalArgumentException Thrown if {@code text} is {@code null}. + */ @Override - public CharSequence normalize (CharSequence text) { - String modified = HASH_USER_REGEX.matcher(text).replaceAll(" "); - modified = RT_REGEX.matcher(modified).replaceAll(" "); - modified = FACE_REGEX.matcher(modified).replaceAll(" "); - modified = LAUGH_REGEX.matcher(modified).replaceAll("$1$2$1$2"); - return modified; + public CharSequence normalize(CharSequence text) { + if (text == null) { + throw new IllegalArgumentException("The text must not be null."); + } + return shrinkLaughter(removeEmoticons(removeRetweetMarkers(removeTagsAndHandles(text)))); + } + + // "[#@]\S+" -> " ": a hash or at sign starts a match only if a non-whitespace char follows; + // the match then swallows the whole non-whitespace run (including further # and @). + private static CharSequence removeTagsAndHandles(CharSequence text) { Review Comment: Done. ########## opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/TwitterCharSequenceNormalizer.java: ########## @@ -16,40 +16,263 @@ */ package opennlp.tools.util.normalizer; -import java.util.regex.Pattern; - /** - * A {@link TwitterCharSequenceNormalizer} implementation that normalizes text + * A {@link CharSequenceNormalizer} implementation that normalizes text * in terms of Twitter character patterns. Every encounter will be replaced by a whitespace. + * + * <p>Four forward cursor passes reproduce, byte for byte, the output of the former regex + * implementation:</p> + * <ol> + * <li>a {@code #} or {@code @} followed by at least one non-whitespace character, together + * with the whole following non-whitespace run, becomes one space (the whitespace class is + * the six ASCII characters the former {@code \S} complemented);</li> + * <li>each maximal sequence of {@code rt} units (case-insensitive, each followed by a space or + * colon) becomes one space when it starts on a word boundary as the JDK regex engine + * defined it for {@code \b};</li> + * <li>each emoticon, eyes {@code :}, {@code ;} or {@code x}, an optional {@code -} nose, and a + * mouth out of {@code ( ) d o p} (case-insensitive), becomes one space, including inside + * words;</li> + * <li>laughter, a run of {@code h}/{@code j}, a run of vowels, and at least one repetition of + * the two runs' last characters, shrinks to those two characters twice + * ({@code "hahaha"} to {@code "haha"}), comparing ASCII case-insensitively.</li> + * </ol> */ public class TwitterCharSequenceNormalizer implements CharSequenceNormalizer { private static final long serialVersionUID = -8155452559337913929L; - - private static final Pattern HASH_USER_REGEX = - Pattern.compile("[#@]\\S+"); - - private static final Pattern RT_REGEX = - Pattern.compile("\\b(rt[ :])+", Pattern.CASE_INSENSITIVE); - - private static final Pattern FACE_REGEX = - Pattern.compile("[:;x]-?[()dop]", Pattern.CASE_INSENSITIVE); - - private static final Pattern LAUGH_REGEX = - Pattern.compile("([hj])+([aieou])+(\\1+\\2+)+", Pattern.CASE_INSENSITIVE); private static final TwitterCharSequenceNormalizer INSTANCE = new TwitterCharSequenceNormalizer(); public static TwitterCharSequenceNormalizer getInstance() { return INSTANCE; } + /** + * @throws IllegalArgumentException Thrown if {@code text} is {@code null}. + */ @Override - public CharSequence normalize (CharSequence text) { - String modified = HASH_USER_REGEX.matcher(text).replaceAll(" "); - modified = RT_REGEX.matcher(modified).replaceAll(" "); - modified = FACE_REGEX.matcher(modified).replaceAll(" "); - modified = LAUGH_REGEX.matcher(modified).replaceAll("$1$2$1$2"); - return modified; + public CharSequence normalize(CharSequence text) { + if (text == null) { + throw new IllegalArgumentException("The text must not be null."); + } + return shrinkLaughter(removeEmoticons(removeRetweetMarkers(removeTagsAndHandles(text)))); + } + + // "[#@]\S+" -> " ": a hash or at sign starts a match only if a non-whitespace char follows; + // the match then swallows the whole non-whitespace run (including further # and @). + private static CharSequence removeTagsAndHandles(CharSequence text) { + final int length = text.length(); + StringBuilder out = null; + int i = 0; + while (i < length) { + final char c = text.charAt(i); + if ((c == '#' || c == '@') + && i + 1 < length && !AsciiChars.WHITESPACE.contains(text.charAt(i + 1))) { + int end = i + 2; + while (end < length && !AsciiChars.WHITESPACE.contains(text.charAt(end))) { + end++; + } + if (out == null) { + out = new StringBuilder(length).append(text, 0, i); + } + out.append(' '); + i = end; + } else { + if (out != null) { + out.append(c); + } + i++; + } + } + return out == null ? text : out.toString(); + } + + // "\b(rt[ :])+" (case-insensitive) -> " ": one or more three-char units of r, t, and a space + // or colon, starting where the JDK engine put a word boundary. + private static CharSequence removeRetweetMarkers(CharSequence text) { Review Comment: Done. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
