This is an automated email from the ASF dual-hosted git repository.
tballison pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tika.git
The following commit(s) were added to refs/heads/main by this push:
new fcdf187e3d TIKA-5061: let commonmark size the code fence, so a three
backtick li… (#3568)
fcdf187e3d is described below
commit fcdf187e3d6e5177ecb98c0ab624b6a7b7190fd9
Author: Tim Allison <[email protected]>
AuthorDate: Sat Nov 7 17:39:27 2026 -0500
TIKA-5061: let commonmark size the code fence, so a three backtick li…
(#3568)
* TIKA-5061: let commonmark size the code fence, so a three backtick line
in <pre> cannot close the block
* TIKA-5061: escape a leading + in markdown text; flatten nesting past 64
levels
---
CHANGES.txt | 8 ++
.../apache/tika/sax/FastMarkdownTextRenderer.java | 14 ++-
.../apache/tika/sax/ToMarkdownContentHandler.java | 60 +++++++--
.../tika/sax/FastMarkdownTextRendererTest.java | 13 +-
.../tika/sax/ToMarkdownContentHandlerTest.java | 139 +++++++++++++++++++++
5 files changed, 222 insertions(+), 12 deletions(-)
diff --git a/CHANGES.txt b/CHANGES.txt
index def32a6c9c..716c93528a 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -104,6 +104,14 @@ Release 4.3.0 - ???
EXTRACTION AND FORMATS
+ * Markdown output keeps document code inside its code spans and blocks: a
line of
+ backticks inside <pre> no longer closes the fenced code block, and a line
break inside
+ inline code, alt text or a link destination, or a lone CR inside <pre>,
no longer starts
+ a new markdown line, so none of them can turn the text after it into raw
HTML. A
+ paragraph starting with "+ " stays a paragraph instead of becoming a
list, and
+ elements nested more than 64 levels deep are flattened, so deep nesting
can no longer
+ overflow the stack or make the output grow with the square of the depth
(TIKA-5061).
+
* HTML main-content extraction: tika-readability is a jsoup-only Java port
of Mozilla
Readability, and "readability-jsoup-filter" in the parse context runs it
in JSoupParser
(off by default; HTML documents only, never message bodies or other HTML
inside a
diff --git
a/tika-core/src/main/java/org/apache/tika/sax/FastMarkdownTextRenderer.java
b/tika-core/src/main/java/org/apache/tika/sax/FastMarkdownTextRenderer.java
index e821ff35c9..f0d5f0545b 100644
--- a/tika-core/src/main/java/org/apache/tika/sax/FastMarkdownTextRenderer.java
+++ b/tika-core/src/main/java/org/apache/tika/sax/FastMarkdownTextRenderer.java
@@ -42,9 +42,11 @@ import org.commonmark.text.AsciiMatcher;
* The escaping semantics replicate {@code
CoreMarkdownNodeRenderer#visit(Text)} for
* the commonmark version pinned in tika-parent exactly: the line-start
disambiguation
* cases, the heading escape variant, the {@code !}-before-link case, and the
- * {@code \n} numeric reference. {@code FastMarkdownTextRendererTest} renders
ASTs
- * through this renderer and the stock one and requires byte-identical output;
a
- * commonmark upgrade that changes escaping fails there and must be re-synced
here.
+ * {@code \n} numeric reference. The one deliberate difference: a {@code +} at
line
+ * start is escaped, since it is a bullet marker the stock renderer misses.
+ * {@code FastMarkdownTextRendererTest} renders ASTs through this renderer and
the
+ * stock one and requires byte-identical output otherwise; a commonmark
upgrade that
+ * changes escaping fails there and must be re-synced here.
* <p>
* Text inside a table cell renders through the stock per-char path: the
tables extension
* pushes a raw-escape for {@code |} onto the writer there, and hand-emitted
escapes would
@@ -115,6 +117,12 @@ class FastMarkdownTextRenderer implements NodeRenderer {
literal = literal.substring(1);
break;
}
+ case '+': {
+ // a bullet marker too; commonmark 0.31.0 leaves it
unescaped
+ writer.raw("\\+");
+ literal = literal.substring(1);
+ break;
+ }
case '=': {
if (text.getPrevious() != null) {
writer.raw("\\=");
diff --git
a/tika-core/src/main/java/org/apache/tika/sax/ToMarkdownContentHandler.java
b/tika-core/src/main/java/org/apache/tika/sax/ToMarkdownContentHandler.java
index 00c7b95d1a..804042012e 100644
--- a/tika-core/src/main/java/org/apache/tika/sax/ToMarkdownContentHandler.java
+++ b/tika-core/src/main/java/org/apache/tika/sax/ToMarkdownContentHandler.java
@@ -25,6 +25,7 @@ import java.io.Writer;
import java.util.ArrayDeque;
import java.util.Arrays;
import java.util.Deque;
+import java.util.Iterator;
import java.util.List;
import java.util.Locale;
@@ -133,6 +134,13 @@ public class ToMarkdownContentHandler extends
DefaultHandler {
// <script>/<style> content is dropped.
private int suppressDepth;
+ /**
+ * Deepest open-node stack that still nests: commonmark renders
recursively (a stack
+ * overflow at a few thousand levels) and prefixes every line with its
containers, so
+ * output would grow with the square of the depth. Past it, content joins
the current node.
+ */
+ static final int MAX_NESTING = 64;
+
// True once endDocument has rendered the full document to the writer.
private boolean finished;
@@ -180,12 +188,22 @@ public class ToMarkdownContentHandler extends
DefaultHandler {
openBlock(new Paragraph());
break;
case "blockquote":
- openBlock(new BlockQuote());
+ popToBlockHost();
+ if (!flattenAtCap()) {
+ openBlock(new BlockQuote());
+ }
break;
case "ul":
- openBlock(newBulletList());
+ popToBlockHost();
+ if (!flattenAtCap()) {
+ openBlock(newBulletList());
+ }
break;
case "ol":
+ popToBlockHost();
+ if (flattenAtCap()) {
+ break;
+ }
OrderedList ol = new OrderedList();
ol.setMarkerStartNumber(1);
ol.setMarkerDelimiter(".");
@@ -198,8 +216,8 @@ public class ToMarkdownContentHandler extends
DefaultHandler {
case "pre":
popToBlockHost();
codeBlock = new FencedCodeBlock();
+ // no fence length: commonmark makes the fence longer than any
backtick run inside
codeBlock.setFenceCharacter("`");
- codeBlock.setOpeningFenceLength(3);
codeBlock.setLiteral("");
stack.peek().appendChild(codeBlock);
codeBlockText = new StringBuilder();
@@ -345,8 +363,9 @@ public class ToMarkdownContentHandler extends
DefaultHandler {
closeImplicitParagraph();
Node popped = stack.size() > 1 ? stack.peek() : null;
pop();
- if (popped instanceof Emphasis || popped instanceof
StrongEmphasis
- || popped instanceof Strikethrough) {
+ // a flattened duplicate: the node is still open below
+ if (popped != stack.peek() && (popped instanceof Emphasis
+ || popped instanceof StrongEmphasis || popped
instanceof Strikethrough)) {
trimDelimited(popped);
}
break;
@@ -609,6 +628,10 @@ public class ToMarkdownContentHandler extends
DefaultHandler {
* implicit bullet list so the model stays renderable.
*/
private void openListItem() {
+ // an item of an open list always opens, or its text would be dropped
+ if (!(stack.peek() instanceof ListBlock) && flattenAtCap()) {
+ return;
+ }
if (!(stack.peek() instanceof ListBlock)) {
popToBlockHost();
BulletList implicit = newBulletList();
@@ -622,10 +645,31 @@ public class ToMarkdownContentHandler extends
DefaultHandler {
private void openInline(Node node) {
ensureInlineContainer();
+ if (flattenAtCap()) {
+ return;
+ }
stack.peek().appendChild(node);
stack.push(node);
}
+ /**
+ * At {@link #MAX_NESTING}, pushes the current node again so the element's
content joins
+ * it and the element's end pops the duplicate.
+ */
+ private boolean flattenAtCap() {
+ if (stack.size() < MAX_NESTING) {
+ return false;
+ }
+ stack.push(stack.peek());
+ return true;
+ }
+
+ private boolean topIsDuplicate() {
+ Iterator<Node> it = stack.iterator();
+ Node top = it.next();
+ return it.hasNext() && it.next() == top;
+ }
+
/**
* Markdown delimiters cannot sit next to whitespace inside them ({@code
** x**} is
* literal), so moves edge whitespace outside and drops the node if
nothing is left.
@@ -678,7 +722,7 @@ public class ToMarkdownContentHandler extends
DefaultHandler {
private void pop() {
if (stack.size() > 1) {
Node popped = stack.pop();
- if (popped == implicitParagraph) {
+ if (popped == implicitParagraph && stack.peek() != popped) {
implicitParagraph = null;
}
}
@@ -688,7 +732,7 @@ public class ToMarkdownContentHandler extends
DefaultHandler {
private void popToBlockHost() {
while (stack.size() > 1 && !canHostBlocks(stack.peek())) {
Node popped = stack.pop();
- if (popped == implicitParagraph) {
+ if (popped == implicitParagraph && stack.peek() != popped) {
implicitParagraph = null;
}
}
@@ -709,7 +753,7 @@ public class ToMarkdownContentHandler extends
DefaultHandler {
}
private void closeImplicitParagraph() {
- if (implicitParagraph != null && stack.peek() == implicitParagraph) {
+ if (implicitParagraph != null && stack.peek() == implicitParagraph &&
!topIsDuplicate()) {
stack.pop();
implicitParagraph = null;
}
diff --git
a/tika-core/src/test/java/org/apache/tika/sax/FastMarkdownTextRendererTest.java
b/tika-core/src/test/java/org/apache/tika/sax/FastMarkdownTextRendererTest.java
index 9122e84269..39412d3063 100644
---
a/tika-core/src/test/java/org/apache/tika/sax/FastMarkdownTextRendererTest.java
+++
b/tika-core/src/test/java/org/apache/tika/sax/FastMarkdownTextRendererTest.java
@@ -82,7 +82,7 @@ public class FastMarkdownTextRendererTest {
public void testLineStartEscapes() {
// Literals whose first char would parse as block structure at line
start
for (String literal : new String[]{
- "- not a list", "-not a list", "-", "+ plus", "* star",
+ "- not a list", "-not a list", "-", "* star",
"# not a heading", "#hash", "#",
"= not setext", "=",
"1. not ordered", "12. also not", "123456789. limit", "1)
paren",
@@ -93,6 +93,17 @@ public class FastMarkdownTextRendererTest {
}
}
+ @Test
+ public void testPlusAtLineStartIsEscaped() {
+ // deliberate divergence: stock commonmark lets "+ x" become a bullet
list
+ for (String literal : new String[]{"+ plus", "+", "+1"}) {
+ String md = FAST.render(paragraphs(literal));
+ assertTrue(md.startsWith("\\+"), md);
+ assertTrue(PARSER.parse(md).getFirstChild() instanceof Paragraph,
md);
+ }
+ assertSameRendering(paragraphs("mid ", "+ plus"));
+ }
+
@Test
public void testSecondTextNodeNotAtLineStart() {
// The line-start branches must not fire mid-line
diff --git
a/tika-core/src/test/java/org/apache/tika/sax/ToMarkdownContentHandlerTest.java
b/tika-core/src/test/java/org/apache/tika/sax/ToMarkdownContentHandlerTest.java
index 6d43be4656..9cdacff20e 100644
---
a/tika-core/src/test/java/org/apache/tika/sax/ToMarkdownContentHandlerTest.java
+++
b/tika-core/src/test/java/org/apache/tika/sax/ToMarkdownContentHandlerTest.java
@@ -440,6 +440,145 @@ public class ToMarkdownContentHandlerTest {
assertTrue(result.contains("\n```"));
}
+ @Test
+ public void testFenceInCodeBlockCannotCloseIt() throws Exception {
+ for (String container : new String[]{"", "li", "blockquote"}) {
+ for (String fence : new String[]{"```", "````", " ```",
"```js"}) {
+ ToMarkdownContentHandler handler = new
ToMarkdownContentHandler();
+ handler.startDocument();
+ if (container.equals("li")) {
+ startElement(handler, "ul");
+ }
+ if (!container.isEmpty()) {
+ startElement(handler, container);
+ }
+ startElement(handler, "pre");
+ chars(handler, "a\n" + fence + "\n<img src=x
onerror=alert(1)>\n");
+ endElement(handler, "pre");
+ if (!container.isEmpty()) {
+ endElement(handler, container);
+ }
+ if (container.equals("li")) {
+ endElement(handler, "ul");
+ }
+ handler.endDocument();
+
+ String md = handler.toString();
+ String html = renderHtml(md);
+ assertFalse(html.contains("<img"), md);
+ assertContains("<img src=x onerror=alert(1)>", html);
+ }
+ }
+ }
+
+ @Test
+ public void testPlusAtLineStartIsNotAList() throws Exception {
+ for (String container : new String[]{"p", "li", "blockquote"}) {
+ ToMarkdownContentHandler handler = new ToMarkdownContentHandler();
+ handler.startDocument();
+ if (container.equals("li")) {
+ startElement(handler, "ul");
+ }
+ startElement(handler, container);
+ chars(handler, "+ not a list");
+ endElement(handler, container);
+ if (container.equals("li")) {
+ endElement(handler, "ul");
+ }
+ handler.endDocument();
+
+ String md = handler.toString();
+ String html = renderHtml(md);
+ assertContains("+ not a list", html);
+ assertFalse(html.contains("<li>not a list"), md);
+ }
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {"blockquote", "ul", "b", "s", "a"})
+ public void testDeepNestingIsCappedWithoutLosingText(String element)
throws Exception {
+ int depth = 20_000;
+ ToMarkdownContentHandler handler = new ToMarkdownContentHandler();
+ handler.startDocument();
+ boolean inline = !element.equals("blockquote") &&
!element.equals("ul");
+ if (inline) {
+ startElement(handler, "p");
+ }
+ for (int i = 0; i < depth; i++) {
+ startElement(handler, element);
+ if (element.equals("ul")) {
+ startElement(handler, "li");
+ }
+ chars(handler, "w" + i + " ");
+ }
+ for (int i = 0; i < depth; i++) {
+ if (element.equals("ul")) {
+ endElement(handler, "li");
+ }
+ endElement(handler, element);
+ }
+ if (inline) {
+ endElement(handler, "p");
+ }
+ chars(handler, "after");
+ handler.endDocument();
+
+ String md = handler.toString();
+ // the cap keeps each line's container prefix short: output stays
linear in the input
+ assertTrue(md.length() < depth * 400, "markdown length " +
md.length());
+ String text = renderHtml(md);
+ for (int i : new int[]{0, 63, 64, 19_999}) {
+ assertContains("w" + i, text);
+ }
+ assertContains("after", text);
+ }
+
+ @Test
+ public void testEmptyFlattenedInlineKeepsLaterText() throws Exception {
+ ToMarkdownContentHandler handler = new ToMarkdownContentHandler();
+ handler.startDocument();
+ startElement(handler, "p");
+ int depth = ToMarkdownContentHandler.MAX_NESTING + 6;
+ for (int i = 0; i < depth; i++) {
+ startElement(handler, "b");
+ }
+ endElement(handler, "b");
+ chars(handler, "kept");
+ for (int i = 1; i < depth; i++) {
+ endElement(handler, "b");
+ }
+ endElement(handler, "p");
+ handler.endDocument();
+
+ assertContains("kept", handler.toString());
+ }
+
+ @Test
+ public void testFlattenedInlineDoesNotSplitParagraph() throws Exception {
+ // sweeps the depths at which the cap lands right on the implicit
paragraph
+ for (int depth = ToMarkdownContentHandler.MAX_NESTING - 6;
+ depth <= ToMarkdownContentHandler.MAX_NESTING + 6; depth++) {
+ ToMarkdownContentHandler handler = new ToMarkdownContentHandler();
+ handler.startDocument();
+ for (int i = 0; i < depth; i++) {
+ startElement(handler, "blockquote");
+ }
+ chars(handler, "x ");
+ startElement(handler, "b");
+ chars(handler, "z");
+ endElement(handler, "b");
+ chars(handler, " y");
+ for (int i = 0; i < depth; i++) {
+ endElement(handler, "blockquote");
+ }
+ handler.endDocument();
+
+ String html = renderHtml(handler.toString());
+ assertTrue(html.matches("(?s).*<p>x (<strong>z</strong>|z)
y</p>.*"),
+ "depth " + depth + ": " + html);
+ }
+ }
+
@Test
public void testInlineCode() throws Exception {
ToMarkdownContentHandler handler = new ToMarkdownContentHandler();
