This is an automated email from the ASF dual-hosted git repository.
rzo1 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/opennlp.git
The following commit(s) were added to refs/heads/main by this push:
new e286928a1 GH-1458: Parse name-finder tags and parse-tree strings
without regular expressions (#1459)
e286928a1 is described below
commit e286928a162a8097ced8c11fe9c433843de8fc3e
Author: Richard Zowalla <[email protected]>
AuthorDate: Fri Oct 9 19:51:20 2026 +0200
GH-1458: Parse name-finder tags and parse-tree strings without regular
expressions (#1459)
* GH-1458: Parse name-finder tags and parse-tree strings without regular
expressions
* GH-1458: Name the separator characters of start tags and parse strings
Replaces the character literals in the new scans with named constants and
reuses
UnicodeWhitespace.SPACE and CARRIAGE_RETURN in StringUtil.isAsciiWhitespace.
* GH-1458: Split the start tag and constituent oracle tests by outcome
Valid inputs, the rejected empty type and invalid inputs are now separate
tests with
explicit expected values; the former regular expressions remain as
cross-checks.
* GH-1458: Use TAG_END for the closing character of a start tag
* GH-1458: Phrase the start tag check as Checks whether
---
.../java/opennlp/tools/namefind/NameSample.java | 50 +++++-
.../src/main/java/opennlp/tools/parser/Parse.java | 109 +++++++++----
.../main/java/opennlp/tools/util/StringUtil.java | 16 ++
.../java/opennlp/tools/util/StringUtilTest.java | 26 ++-
.../opennlp/tools/namefind/NameSampleTest.java | 67 ++++++++
.../test/java/opennlp/tools/parser/ParseTest.java | 181 +++++++++++++++++++++
6 files changed, 413 insertions(+), 36 deletions(-)
diff --git a/opennlp-api/src/main/java/opennlp/tools/namefind/NameSample.java
b/opennlp-api/src/main/java/opennlp/tools/namefind/NameSample.java
index 4007a4e78..23bf06f32 100644
--- a/opennlp-api/src/main/java/opennlp/tools/namefind/NameSample.java
+++ b/opennlp-api/src/main/java/opennlp/tools/namefind/NameSample.java
@@ -24,13 +24,12 @@ import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Objects;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
import opennlp.tools.commons.Sample;
import opennlp.tools.tokenize.WhitespaceTokenizer;
import opennlp.tools.util.ParamChecks;
import opennlp.tools.util.Span;
+import opennlp.tools.util.StringUtil;
/**
* Encapsulates names for a single unit of text.
@@ -44,6 +43,12 @@ public class NameSample implements Sample {
public static final String START_TAG = "<START>";
public static final String END_TAG = "<END>";
+ /** Separates {@code <START} from the name type in a start tag. */
+ private static final char TYPE_SEPARATOR = ':';
+
+ /** Ends a start tag. */
+ private static final char TAG_END = '>';
+
private final String id;
private final List<String> sentence;
@@ -266,8 +271,42 @@ public class NameSample implements Sample {
return errorString.toString();
}
- private static final Pattern START_TAG_PATTERN =
Pattern.compile("<START(:([^:>\\s]*))?>");
+ /**
+ * Checks whether a token is a start tag, either {@code <START>} or {@code
<START:type>}, where
+ * the possibly empty type contains neither {@code ':'}, {@code '>'} nor
ASCII whitespace.
+ *
+ * @param token The token to test.
+ * @return {@code true} if {@code token} is a start tag.
+ */
+ private static boolean isStartTag(String token) {
+ if (START_TAG.equals(token)) {
+ return true;
+ }
+ if (!token.startsWith(START_TAG_PREFIX) || token.length() <=
START_TAG_PREFIX.length()
+ || token.charAt(token.length() - 1) != TAG_END) {
+ return false;
+ }
+ for (int i = START_TAG_PREFIX.length(); i < token.length() - 1; i++) {
+ char c = token.charAt(i);
+ if (c == TYPE_SEPARATOR || c == TAG_END ||
StringUtil.isAsciiWhitespace(c)) {
+ return false;
+ }
+ }
+ return true;
+ }
+ /**
+ * Extracts the type of a start tag accepted by {@link #isStartTag(String)}.
+ *
+ * @param startTag A start tag.
+ * @return The type, which is empty for {@code <START:>}, or {@code null}
for {@code <START>}.
+ */
+ private static String startTagType(String startTag) {
+ if (START_TAG.equals(startTag)) {
+ return null;
+ }
+ return startTag.substring(START_TAG_PREFIX.length(), startTag.length() -
1);
+ }
/**
* Parses given input into a {@link NameSample}.
@@ -310,15 +349,14 @@ public class NameSample implements Sample {
boolean catchingName = false;
for (int pi = 0; pi < parts.length; pi++) {
- Matcher startMatcher = START_TAG_PATTERN.matcher(parts[pi]);
- if (startMatcher.matches()) {
+ if (isStartTag(parts[pi])) {
if (catchingName) {
throw new IOException("Found unexpected annotation" +
" while handling a name sequence: " +
errorTokenWithContext(parts, pi));
}
catchingName = true;
startIndex = wordIndex;
- String nameTypeFromSample = startMatcher.group(2);
+ String nameTypeFromSample = startTagType(parts[pi]);
if (nameTypeFromSample != null) {
if (nameTypeFromSample.length() == 0) {
throw new IOException("Missing a name type: " +
errorTokenWithContext(parts, pi));
diff --git a/opennlp-api/src/main/java/opennlp/tools/parser/Parse.java
b/opennlp-api/src/main/java/opennlp/tools/parser/Parse.java
index f4e3c2f53..dc921f837 100644
--- a/opennlp-api/src/main/java/opennlp/tools/parser/Parse.java
+++ b/opennlp-api/src/main/java/opennlp/tools/parser/Parse.java
@@ -27,14 +27,13 @@ import java.util.Objects;
import java.util.Set;
import java.util.Stack;
import java.util.TreeSet;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import opennlp.tools.util.ParamChecks;
import opennlp.tools.util.Span;
+import opennlp.tools.util.StringUtil;
/**
* Data structure for holding parse constituents.
@@ -49,6 +48,21 @@ public class Parse implements Cloneable, Comparable<Parse> {
public static final String BRACKET_LSB = "[";
public static final String BRACKET_RSB = "]";
+ /** Opens a constituent in the bracketed parse string. */
+ private static final char OPENING_BRACKET = BRACKET_LRB.charAt(0);
+
+ /** Closes a constituent in the bracketed parse string. */
+ private static final char CLOSING_BRACKET = BRACKET_RRB.charAt(0);
+
+ /** Separates a constituent label from its token. */
+ private static final char LABEL_SEPARATOR = ' ';
+
+ /** Separates a constituent type from its function tag, as in {@code
NP-SBJ}. */
+ private static final char FUNCTION_TAG_SEPARATOR = '-';
+
+ /** Separates a constituent label from its co-index, as in {@code NP=2}. */
+ private static final char INDEX_SEPARATOR = '=';
+
/**
* The text string on which this parse is based.
* This object is shared among all parses for the same sentence.
@@ -107,22 +121,6 @@ public class Parse implements Cloneable, Comparable<Parse>
{
*/
private boolean isChunk;
- /**
- * The pattern used to find the base constituent label of a
- * Penn Treebank labeled constituent.
- */
- private static final Pattern typePattern = Pattern.compile("^([^ =-]+)");
-
- /**
- * The pattern used to find the function tags.
- */
- private static final Pattern funTypePattern = Pattern.compile("^[^ =-]+-([^
=-]+)");
-
- /**
- * The patter used to identify tokens in Penn Treebank labeled constituents.
- */
- private static final Pattern tokenPattern = Pattern.compile("^[^ ()]+ ([^
()]+)\\s*\\)");
-
/**
* The set of punctuation parses which are between this parse and the
previous parse.
*/
@@ -669,14 +667,13 @@ public class Parse implements Cloneable,
Comparable<Parse> {
} else if (rest.startsWith("-NONE-")) {
return "-NONE-";
} else {
- Matcher typeMatcher = typePattern.matcher(rest);
- if (typeMatcher.find()) {
- String type = typeMatcher.group(1);
- if (useFunctionTags) {
- Matcher funMatcher = funTypePattern.matcher(rest);
- if (funMatcher.find()) {
- String ftag = funMatcher.group(1);
- type = type + "-" + ftag;
+ int typeEnd = endOfLabel(rest, 0);
+ if (typeEnd > 0) {
+ String type = rest.substring(0, typeEnd);
+ if (useFunctionTags && typeEnd < rest.length() && rest.charAt(typeEnd)
== FUNCTION_TAG_SEPARATOR) {
+ int ftagEnd = endOfLabel(rest, typeEnd + 1);
+ if (ftagEnd > typeEnd + 1) {
+ type = type + FUNCTION_TAG_SEPARATOR + rest.substring(typeEnd + 1,
ftagEnd);
}
}
return type;
@@ -685,6 +682,47 @@ public class Parse implements Cloneable, Comparable<Parse>
{
return null;
}
+ /**
+ * Finds the end of the run of label characters that starts at {@code from}.
A label
+ * character is any character except space, {@code '='} and {@code '-'},
which separate a
+ * constituent label from its function tags and co-index.
+ *
+ * @param text The text to scan.
+ * @param from The offset the run starts at.
+ * @return The offset after the last label character, or {@code from} if
there is none.
+ */
+ private static int endOfLabel(String text, int from) {
+ int i = from;
+ while (i < text.length()) {
+ char c = text.charAt(i);
+ if (c == LABEL_SEPARATOR || c == INDEX_SEPARATOR || c ==
FUNCTION_TAG_SEPARATOR) {
+ break;
+ }
+ i++;
+ }
+ return i;
+ }
+
+ /**
+ * Finds the end of the run of token characters that starts at {@code from}.
A token
+ * character is any character except space and parentheses.
+ *
+ * @param text The text to scan.
+ * @param from The offset the run starts at.
+ * @return The offset after the last token character, or {@code from} if
there is none.
+ */
+ private static int endOfTokenText(String text, int from) {
+ int i = from;
+ while (i < text.length()) {
+ char c = text.charAt(i);
+ if (c == LABEL_SEPARATOR || c == OPENING_BRACKET || c ==
CLOSING_BRACKET) {
+ break;
+ }
+ i++;
+ }
+ return i;
+ }
+
private static String encodeToken(String token) {
if (BRACKET_LRB.equals(token)) {
return "-LRB-";
@@ -727,9 +765,22 @@ public class Parse implements Cloneable, Comparable<Parse>
{
* string or {@code null} if the portion of the parse string does not
represent a token.
*/
private static String getToken(String rest) {
- Matcher tokenMatcher = tokenPattern.matcher(rest);
- if (tokenMatcher.find()) {
- return decodeToken(tokenMatcher.group(1));
+ // Expected form: type, one space, token, optional ASCII whitespace,
closing parenthesis.
+ int typeEnd = endOfTokenText(rest, 0);
+ if (typeEnd == 0 || typeEnd == rest.length() || rest.charAt(typeEnd) !=
LABEL_SEPARATOR) {
+ return null;
+ }
+ int tokenStart = typeEnd + 1;
+ int tokenEnd = endOfTokenText(rest, tokenStart);
+ if (tokenEnd == tokenStart) {
+ return null;
+ }
+ int i = tokenEnd;
+ while (i < rest.length() && StringUtil.isAsciiWhitespace(rest.charAt(i))) {
+ i++;
+ }
+ if (i < rest.length() && rest.charAt(i) == CLOSING_BRACKET) {
+ return decodeToken(rest.substring(tokenStart, tokenEnd));
}
return null;
}
diff --git a/opennlp-api/src/main/java/opennlp/tools/util/StringUtil.java
b/opennlp-api/src/main/java/opennlp/tools/util/StringUtil.java
index 72d34f620..6b7839d10 100644
--- a/opennlp-api/src/main/java/opennlp/tools/util/StringUtil.java
+++ b/opennlp-api/src/main/java/opennlp/tools/util/StringUtil.java
@@ -31,6 +31,9 @@ public class StringUtil {
private static final char BYTE_ORDER_MARK = '\uFEFF';
+ /** U+0009, the first of the contiguous ASCII whitespace controls U+0009 to
U+000D. */
+ private static final int CHARACTER_TABULATION = 0x0009;
+
private static final Logger logger =
LoggerFactory.getLogger(StringUtil.class);
/**
@@ -383,6 +386,19 @@ public class StringUtil {
return codePoint >= '0' && codePoint <= '9';
}
+ /**
+ * Tests for an ASCII whitespace character: space, tab, line feed, vertical
tab, form feed
+ * or carriage return. This is the set matched by {@code \s} in a
+ * {@link java.util.regex.Pattern} compiled without {@code
UNICODE_CHARACTER_CLASS}.
+ *
+ * @param codePoint The code point to test.
+ * @return {@code true} if {@code codePoint} is an ASCII whitespace
character.
+ */
+ public static boolean isAsciiWhitespace(int codePoint) {
+ return codePoint == UnicodeWhitespace.SPACE
+ || (codePoint >= CHARACTER_TABULATION && codePoint <=
UnicodeWhitespace.CARRIAGE_RETURN);
+ }
+
/**
* Finds the end of the run of ASCII digits that starts at {@code from}.
*
diff --git a/opennlp-api/src/test/java/opennlp/tools/util/StringUtilTest.java
b/opennlp-api/src/test/java/opennlp/tools/util/StringUtilTest.java
index e71ab8675..1aa05bb8b 100644
--- a/opennlp-api/src/test/java/opennlp/tools/util/StringUtilTest.java
+++ b/opennlp-api/src/test/java/opennlp/tools/util/StringUtilTest.java
@@ -20,6 +20,7 @@ package opennlp.tools.util;
import java.nio.CharBuffer;
import java.util.Arrays;
import java.util.List;
+import java.util.regex.Pattern;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
import java.util.stream.Stream;
@@ -202,7 +203,7 @@ public class StringUtilTest {
}
// -------------------------------------------------------------------------
- // isAsciiLetter, isAsciiLowerCase, isAsciiDigit, endOfAsciiDigits
+ // isAsciiLetter, isAsciiLowerCase, isAsciiDigit, isAsciiWhitespace,
endOfAsciiDigits
// -------------------------------------------------------------------------
@ParameterizedTest
@@ -246,6 +247,29 @@ public class StringUtilTest {
Assertions.assertFalse(StringUtil.isAsciiDigit(codePoint));
}
+ @ParameterizedTest
+ @ValueSource(ints = {' ', '\t', '\n', 0x0B, '\f', '\r'})
+ void testIsAsciiWhitespaceAccepts(int codePoint) {
+ Assertions.assertTrue(StringUtil.isAsciiWhitespace(codePoint));
+ }
+
+ @ParameterizedTest
+ // the neighbors of the range, the information separators, NEL, no-break
space,
+ // ideographic space, line separator
+ @ValueSource(ints = {0x08, 0x0E, 0x1C, 0x1F, '!', 0x85, 0xA0, 0x3000,
0x2028, 0, -1})
+ void testIsAsciiWhitespaceRejects(int codePoint) {
+ Assertions.assertFalse(StringUtil.isAsciiWhitespace(codePoint));
+ }
+
+ @Test
+ void testIsAsciiWhitespaceMatchesRegexClass() {
+ Pattern whitespace = Pattern.compile("\\s");
+ for (int cp = 0; cp <= Character.MAX_CODE_POINT; cp++) {
+
Assertions.assertEquals(whitespace.matcher(Character.toString(cp)).matches(),
+ StringUtil.isAsciiWhitespace(cp), Integer.toHexString(cp));
+ }
+ }
+
@ParameterizedTest
@MethodSource("asciiDigitRuns")
void testEndOfAsciiDigits(String text, int from, int expected) {
diff --git
a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/namefind/NameSampleTest.java
b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/namefind/NameSampleTest.java
index 8ea8b879e..1a0526f3d 100644
---
a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/namefind/NameSampleTest.java
+++
b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/namefind/NameSampleTest.java
@@ -24,9 +24,13 @@ import java.io.ObjectInput;
import java.io.ObjectInputStream;
import java.io.ObjectOutput;
import java.io.ObjectOutputStream;
+import java.util.regex.Pattern;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.CsvSource;
+import org.junit.jupiter.params.provider.ValueSource;
import opennlp.tools.util.Span;
@@ -302,6 +306,69 @@ public class NameSampleTest {
return createSimpleNameSample(false);
}
+ /**
+ * The regular expression that matched start tags before 3.0.0. The tests
below assert that
+ * it agrees with the current parser on every input.
+ */
+ private static final Pattern START_TAG_ORACLE =
Pattern.compile("<START(:([^:>\\s]*))?>");
+
+ /**
+ * Checks that a valid start tag opens a name of the given type. A tag
without a type uses
+ * the default type passed to the parser.
+ *
+ * @param tag The start tag.
+ * @param expectedType The expected name type.
+ * @throws IOException Thrown if parsing fails.
+ */
+ @ParameterizedTest
+ @CsvSource(delimiter = '|', value = {
+ "<START>|fallback",
+ "<START:person>|person",
+ "<START:type-1>|type-1",
+ "<START:NP-SBJ=2>|NP-SBJ=2",
+ "<START:a.b_c/d;e,f&g%h$i>|a.b_c/d;e,f&g%h$i",
+ "<START:a\u001Cb>|a\u001Cb",
+ "<START:\u00E9t\u00E9>|\u00E9t\u00E9",
+ "<START:\uD83D\uDE00>|\uD83D\uDE00"})
+ void testValidStartTagOpensName(String tag, String expectedType) throws
IOException {
+ Assertions.assertTrue(START_TAG_ORACLE.matcher(tag).matches());
+
+ NameSample sample = NameSample.parse(tag + " word <END>", "fallback",
false);
+ Assertions.assertArrayEquals(new String[] {"word"}, sample.getSentence());
+ Assertions.assertArrayEquals(new Span[] {new Span(0, 1, expectedType)},
sample.getNames());
+ }
+
+ /**
+ * Checks that a start tag with an empty type is rejected.
+ */
+ @Test
+ void testStartTagWithEmptyTypeIsRejected() {
+ Assertions.assertTrue(START_TAG_ORACLE.matcher("<START:>").matches());
+
+ Assertions.assertThrows(IOException.class,
+ () -> NameSample.parse("<START:> word <END>", false));
+ }
+
+ /**
+ * Checks that a token which is not a valid start tag is kept as an ordinary
token and does
+ * not open a name.
+ *
+ * @param token The token that resembles a start tag.
+ * @throws IOException Thrown if parsing fails.
+ */
+ @ParameterizedTest
+ @ValueSource(strings = {
+ "<START:a:b>", "<START::>", "<START:a>b>", "<START:a>>", "<START>>",
"<START>a>",
+ "<START", "<START:", "<START:person", "START:person>", "<start:person>",
"<STARTX>",
+ "<START;person>", "x<START>", "<START:person>x", "<END:x>"})
+ void testInvalidStartTagIsKeptAsToken(String token) throws IOException {
+ Assertions.assertFalse(START_TAG_ORACLE.matcher(token).matches());
+
+ NameSample sample = NameSample.parse(token + " word", false);
+ Assertions.assertArrayEquals(new String[] {token, "word"},
sample.getSentence());
+ Assertions.assertEquals(0, sample.getNames().length);
+ }
+
@Test
void testRejectsNullSentence() {
IllegalArgumentException e =
Assertions.assertThrows(IllegalArgumentException.class,
diff --git
a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/parser/ParseTest.java
b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/parser/ParseTest.java
index bdf5b6de5..a88f571cc 100644
---
a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/parser/ParseTest.java
+++
b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/parser/ParseTest.java
@@ -17,8 +17,17 @@
package opennlp.tools.parser;
+import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import java.util.stream.Stream;
+
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
+import org.junit.jupiter.params.provider.ValueSource;
import opennlp.tools.util.Span;
@@ -176,4 +185,176 @@ public class ParseTest {
Assertions.assertEquals(1, p.getChildren().length);
Assertions.assertEquals(new Span(0, 5), p.getChildren()[0].getSpan());
}
+
+ /**
+ * Reference expressions for the constituent label, the function tag and the
token of a
+ * tree-bank constituent before 3.0.0; the constituent tests below assert
that they agree
+ * with the current parser.
+ */
+ private static final Pattern TYPE_ORACLE = Pattern.compile("^([^ =-]+)");
+ private static final Pattern FUNCTION_TAG_ORACLE = Pattern.compile("^[^
=-]+-([^ =-]+)");
+ private static final Pattern TOKEN_ORACLE = Pattern.compile("^[^ ()]+ ([^
()]+)\\s*\\)");
+
+ private static final Map<String, String> BRACKETS = Map.of("-LRB-", "(",
"-RRB-", ")",
+ "-LCB-", "{", "-RCB-", "}", "-LSB-", "[", "-RSB-", "]");
+
+ /**
+ * Supplies constituents with a valid label: the input after the opening
parenthesis, the
+ * expected label without and with function tags, and the expected token.
+ *
+ * @return The constituents and their expected label and token.
+ */
+ private static Stream<Arguments> constituentLabels() {
+ return Stream.of(
+ Arguments.of("NN word)", "NN", "NN", "word"),
+ Arguments.of("NP-SBJ word)", "NP", "NP-SBJ", "word"),
+ Arguments.of("NP-SBJ=2 word)", "NP", "NP-SBJ", "word"),
+ Arguments.of("NP-SBJ-1 word)", "NP", "NP-SBJ", "word"),
+ Arguments.of("NP-SBJ-TMP=3 word)", "NP", "NP-SBJ", "word"),
+ Arguments.of("NP=2 word)", "NP", "NP", "word"),
+ Arguments.of("NP--X word)", "NP", "NP", "word"),
+ Arguments.of("NP- word)", "NP", "NP", "word"),
+ Arguments.of("PRP$ her)", "PRP$", "PRP$", "her"),
+ Arguments.of("-NONE- *T*-1)", "-NONE-", "-NONE-", "*T*-1"),
+ Arguments.of("-LRB- -LRB-)", "-LRB-", "-LRB-", "("),
+ Arguments.of("-RRB- -RRB-)", "-RRB-", "-RRB-", ")"),
+ Arguments.of("-LCB- -LCB-)", "-LCB-", "-LCB-", "{"),
+ Arguments.of("-RSB- -RSB-)", "-RSB-", "-RSB-", "]"),
+ Arguments.of(". .)", ".", ".", "."),
+ Arguments.of(", ,)", ",", ",", ","),
+ Arguments.of("NN a=b-c)", "NN", "NN", "a=b-c"));
+ }
+
+ /**
+ * Checks the label and token of a valid constituent, without and with
function tags.
+ *
+ * @param rest The constituent text after the opening parenthesis.
+ * @param type The expected label without function tags.
+ * @param typeWithFunctionTag The expected label with function tags.
+ * @param token The expected token.
+ */
+ @ParameterizedTest
+ @MethodSource("constituentLabels")
+ void testConstituentLabel(String rest, String type, String
typeWithFunctionTag, String token) {
+ Assertions.assertEquals(type, oracleType(rest, false));
+ Assertions.assertEquals(typeWithFunctionTag, oracleType(rest, true));
+
+ assertConstituent(rest, false, type, token);
+ assertConstituent(rest, true, typeWithFunctionTag, token);
+ }
+
+ /**
+ * Supplies constituents whose token is followed by whitespace or contains
non-ASCII
+ * characters: the input after the opening parenthesis and the expected
token. Only ASCII
+ * whitespace between the token and the closing parenthesis is skipped.
+ *
+ * @return The constituents and their expected token.
+ */
+ private static Stream<Arguments> constituentTokens() {
+ return Stream.of(
+ Arguments.of("NN word )", "word"),
+ Arguments.of("NN word )", "word"),
+ Arguments.of("NN word \t)", "word"),
+ Arguments.of("NN word \u000B\f\r\n)", "word"),
+ Arguments.of("NN word\t)", "word\t"),
+ Arguments.of("NN word )", "word "),
+ Arguments.of("NN été)", "été"),
+ Arguments.of("NN 😀)", "😀"));
+ }
+
+ /**
+ * Checks the token of a constituent whose token ends in whitespace or
contains non-ASCII
+ * characters.
+ *
+ * @param rest The constituent text after the opening parenthesis.
+ * @param token The expected token.
+ */
+ @ParameterizedTest
+ @MethodSource("constituentTokens")
+ void testConstituentToken(String rest, String token) {
+ Assertions.assertEquals(token, oracleToken(rest));
+
+ assertConstituent(rest, false, "NN", token);
+ assertConstituent(rest, true, "NN", token);
+ }
+
+ /**
+ * Checks that an invalid constituent yields no node, without and with
function tags.
+ *
+ * @param rest The constituent text after the opening parenthesis.
+ */
+ @ParameterizedTest
+ @ValueSource(strings = {
+ "NN word )", "NN word \u001C)", "NN word x)", "NN word)", "NN)", "NN
word",
+ "NN word ", "NN ", "NN"})
+ void testInvalidConstituentYieldsNoNode(String rest) {
+ Assertions.assertNull(oracleToken(rest));
+
+ for (boolean functionTags : new boolean[] {false, true}) {
+ Parse.useFunctionTags(functionTags);
+ try {
+ Assertions.assertEquals(0, Parse.parseParse("(" +
rest).getChildren().length, rest);
+ } finally {
+ Parse.useFunctionTags(false);
+ }
+ }
+ }
+
+ /**
+ * Parses a single constituent and checks that it yields one node with the
given label and
+ * token.
+ *
+ * @param rest The constituent text after the opening parenthesis.
+ * @param functionTags Whether function tags are appended to the label.
+ * @param type The expected label.
+ * @param token The expected token.
+ */
+ private static void assertConstituent(String rest, boolean functionTags,
String type,
+ String token) {
+ Parse.useFunctionTags(functionTags);
+ try {
+ Parse p = Parse.parseParse("(" + rest);
+ Assertions.assertEquals(1, p.getChildren().length, rest);
+ Assertions.assertEquals(type, p.getChildren()[0].getType(), rest);
+ Assertions.assertEquals(token, p.getChildren()[0].getCoveredText(),
rest);
+ } finally {
+ Parse.useFunctionTags(false);
+ }
+ }
+
+ /**
+ * Computes the expected label of a constituent with the reference
expressions.
+ *
+ * @param rest The constituent text after the opening parenthesis.
+ * @param functionTags Whether function tags are appended to the label.
+ * @return The expected label, or {@code null} if there is none.
+ */
+ private static String oracleType(String rest, boolean functionTags) {
+ for (String bracket : new String[] {"-LCB-", "-RCB-", "-LRB-", "-RRB-",
"-RSB-", "-LSB-",
+ "-NONE-"}) {
+ if (rest.startsWith(bracket)) {
+ return bracket;
+ }
+ }
+ Matcher type = TYPE_ORACLE.matcher(rest);
+ if (!type.find()) {
+ return null;
+ }
+ Matcher functionTag = FUNCTION_TAG_ORACLE.matcher(rest);
+ if (functionTags && functionTag.find()) {
+ return type.group(1) + "-" + functionTag.group(1);
+ }
+ return type.group(1);
+ }
+
+ /**
+ * Computes the expected token of a constituent with the reference
expression.
+ *
+ * @param rest The constituent text after the opening parenthesis.
+ * @return The expected decoded token, or {@code null} if there is none.
+ */
+ private static String oracleToken(String rest) {
+ Matcher token = TOKEN_ORACLE.matcher(rest);
+ return token.find() ? BRACKETS.getOrDefault(token.group(1),
token.group(1)) : null;
+ }
}
