This is an automated email from the ASF dual-hosted git repository.
rzo1 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/opennlp.git
The following commit(s) were added to refs/heads/main by this push:
new ce2da5357 GH-1447: Reject self-closing AD markup tags as opening tags
(#1448)
ce2da5357 is described below
commit ce2da53578d2e9113e666acd5818680a2ca247d8
Author: Richard Zowalla <[email protected]>
AuthorDate: Fri Oct 9 09:32:53 2026 +0200
GH-1447: Reject self-closing AD markup tags as opening tags (#1448)
The AD reader no longer treats <s />, <s id="1"/> or any other tag with a
slash right before the closing angle bracket as an opening tag, matching
the existing handling of <s/>. Tests pin both sides, and the manual and
README describe the change.
---
README.md | 1 +
.../opennlp/tools/formats/ad/ADSentenceStream.java | 7 ++++---
.../opennlp/tools/formats/conllu/ConlluStream.java | 7 +++----
.../tools/formats/ad/ADSentenceStreamTest.java | 23 ++++++++++++++++++++++
opennlp-docs/src/docbkx/corpora.xml | 9 +++++----
5 files changed, 36 insertions(+), 11 deletions(-)
diff --git a/README.md b/README.md
index 259eb9081..f3af72796 100644
--- a/README.md
+++ b/README.md
@@ -133,6 +133,7 @@ For users of the traditional CLI toolkit, nothing changes
with the 3.x release l
- `AbstractClassPathModelFinder`: the protected `asRegex` and `matchesPattern`
methods were removed. Custom model finders call `matchesWildcard(url, "*.bin")`
instead ([OPENNLP-1932](https://issues.apache.org/jira/browse/OPENNLP-1932)).
- `BasicContextGenerator(String)`: the separator is taken literally instead of
as a regular expression, and a separator containing a backslash throws
`IllegalArgumentException`. Empty predicates are no longer returned, and the
default constructor splits on runs of Unicode whitespace
([OPENNLP-1929](https://issues.apache.org/jira/browse/OPENNLP-1929)).
+- AD reader: a self-closing markup line such as `<s/>`, `<s />` or `<s
id="1"/>` no longer opens a sentence, paragraph, title, box or text. A tag
whose name only starts with `s`, `p`, `t`, `caixa` or `ext`, such as `<sx>`, is
no longer read as that tag, and attributes must be separated from the tag name
by Unicode whitespace
([GH-1447](https://github.com/apache/opennlp/issues/1447)).
- MASC reader: region offsets must be written with ASCII digits only and no
sign. A document with a signed or non-ASCII offset is skipped
([OPENNLP-1990](https://issues.apache.org/jira/browse/OPENNLP-1990)).
### Thread safety
diff --git
a/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/ad/ADSentenceStream.java
b/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/ad/ADSentenceStream.java
index 84d110a09..da3331ea0 100644
---
a/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/ad/ADSentenceStream.java
+++
b/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/ad/ADSentenceStream.java
@@ -969,6 +969,7 @@ public class ADSentenceStream extends
FilterObjectStream<String, ADSentenceStrea
private static final String PARAGRAPH_TAG = "p";
private static final char TAG_OPEN = '<';
private static final char TAG_CLOSE = '>';
+ private static final char SLASH = '/';
private static final String CLOSING_TAG_OPEN = "</";
private final SentenceParser parser;
@@ -1044,8 +1045,8 @@ public class ADSentenceStream extends
FilterObjectStream<String, ADSentenceStrea
* Checks whether a line is an opening markup tag with the given name: the
name right after the
* opening angle bracket, then either the closing angle bracket or
whitespace and attributes,
* which contain no closing angle bracket, then the closing angle bracket as
the last character.
- * A slash directly after the name, as in {@code <s/>}, is rejected. A slash
after whitespace
- * counts as part of the attributes, so {@code <s />} and {@code <s
id="1"/>} are accepted.
+ * A self-closing tag, with a slash right before the closing angle bracket
as in {@code <s/>},
+ * {@code <s />} or {@code <s id="1"/>}, is not an opening tag.
*
* @param line The line.
* @param name The tag name.
@@ -1055,7 +1056,7 @@ public class ADSentenceStream extends
FilterObjectStream<String, ADSentenceStrea
int last = line.length() - 1;
int afterName = name.length() + 1;
if (last < afterName || line.charAt(0) != TAG_OPEN ||
!line.startsWith(name, 1)
- || line.charAt(last) != TAG_CLOSE) {
+ || line.charAt(last) != TAG_CLOSE || line.charAt(last - 1) == SLASH) {
return false;
}
if (afterName < last &&
!StringUtil.isUnicodeWhitespace(line.charAt(afterName))) {
diff --git
a/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/conllu/ConlluStream.java
b/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/conllu/ConlluStream.java
index 8241d98d7..fff73dc7d 100644
---
a/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/conllu/ConlluStream.java
+++
b/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/conllu/ConlluStream.java
@@ -48,7 +48,6 @@ public class ConlluStream implements
ObjectStream<ConlluSentence> {
private static final char MULTIWORD_SEPARATOR = '-';
private static final String INVALID_MULTIWORD_ID = "Invalid multiword token
id: ";
private static final String BACKWARDS_MULTIWORD_ID = "Multiword token id
runs backwards: ";
- private static final String MISSING_MULTIWORD_LINE = "Multiword token %s has
no word line for id %s";
/**
* The token range a multiword token line covers.
@@ -161,8 +160,8 @@ public class ConlluStream implements
ObjectStream<ConlluSentence> {
*
* @param lines The word lines of one sentence.
* @return The lines with each range merged.
- * @throws InvalidFormatException If a multiword token id is malformed, or a
range names a
- * word id that has no line.
+ * @throws InvalidFormatException Thrown if a multiword token id is
malformed, or a range names
+ * a word id that has no line.
*/
private List<ConlluWordLine> postProcessContractions(List<ConlluWordLine>
lines)
throws InvalidFormatException {
@@ -184,7 +183,7 @@ public class ConlluStream implements
ObjectStream<ConlluSentence> {
String js = Long.toString(j);
if (!index.containsKey(js)) {
throw new InvalidFormatException(
- String.format(MISSING_MULTIWORD_LINE, line.getId(), js));
+ String.format("Multiword token %s has no word line for id %s",
line.getId(), js));
}
expandedContractions.add(js);
linesToDelete.add(js);
diff --git
a/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/ad/ADSentenceStreamTest.java
b/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/ad/ADSentenceStreamTest.java
index 737199a1a..204d8fe55 100644
---
a/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/ad/ADSentenceStreamTest.java
+++
b/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/ad/ADSentenceStreamTest.java
@@ -137,6 +137,24 @@ public class ADSentenceStreamTest {
}
}
+ /** A title tag marks the next sentence as a title, a self-closing title tag
does not. */
+ @ParameterizedTest
+ @CsvSource(delimiter = '|', value = {
+ "<t>|true",
+ "<t id=\"1\">|true",
+ "<t/>|false",
+ "<t />|false",
+ "<t id=\"1\"/>|false"
+ })
+ void testTitleTagMarksATitleOnlyWhenOpening(String titleTag, boolean title)
throws IOException {
+ try (ADSentenceStream stream = stream(lines(titleTag, "<s id=\"1\">",
SOURCE, FIRST_TEXT,
+ join(FIRST_TREE), "</s>"))) {
+ ADSentenceStream.Sentence sentence = stream.read();
+ Assertions.assertNotNull(sentence);
+ Assertions.assertEquals(title, sentence.metadata().contains(" title"));
+ }
+ }
+
/** The number of elements of the clause node under the root. */
private static int elementsOfClause(ADSentenceStream.Sentence sentence) {
ADSentenceStream.SentenceParser.TreeElement[] top =
sentence.root().getElements();
@@ -433,6 +451,11 @@ public class ADSentenceStreamTest {
"<t/>|t|false",
"<caixa/>|caixa|false",
"<ext/>|ext|false",
+ "<s />|s|false",
+ "<s id=\"1\"/>|s|false",
+ "<p id=\"1\" />|p|false",
+ "<s id=\"a/b\">|s|true",
+ "<s id=\"1\" >|s|true",
// next line is Unicode whitespace, file separator is not
"<s\u0085id=\"1\">|s|true",
"<s\u001Cid=\"1\">|s|false"
diff --git a/opennlp-docs/src/docbkx/corpora.xml
b/opennlp-docs/src/docbkx/corpora.xml
index da5265901..a1b8bc4bd 100644
--- a/opennlp-docs/src/docbkx/corpora.xml
+++ b/opennlp-docs/src/docbkx/corpora.xml
@@ -480,10 +480,11 @@ Execution time: 1,955 seconds]]>
<code><p></code>, <code><t></code>,
<code><caixa></code>, and
<code><ext></code> may have attributes
after whitespace. A tag with a name that only
starts with one of these names, such as
<code><sx></code>, and a self-closing tag
- such as <code><s/></code> are not
sentence tags. A leaf line of a tree has its
- lexeme after the closing parenthesis, separated
by whitespace; a line with no text or
- only whitespace after the parenthesis has no
lexeme. Inside a quoted lemma, a secondary
- tag, or the lexeme, a line separator or next
line character is kept as part of the text.
+ such as <code><s/></code> or <code><s
id="1"/></code> are not sentence tags.
+ A leaf line of a tree has its lexeme after the
closing parenthesis, separated by
+ whitespace; a line with no text or only
whitespace after the parenthesis has no lexeme.
+ Inside a quoted lemma, a secondary tag, or the
lexeme, a line separator or next line
+ character is kept as part of the text.
A tree line that starts with equals signs but
has no tag keeps all of the text after the
equals signs as its lexeme. Such a line is
skipped when that text is empty, or when it is
a word of any script followed by a period or an
angle bracket, and a line separator inside
