This is an automated email from the ASF dual-hosted git repository.
tballison pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tika.git
The following commit(s) were added to refs/heads/main by this push:
new ceaa982d70 TIKA-4759 - use a markdown writer (#2900)
ceaa982d70 is described below
commit ceaa982d704ae478879b679bdf89a05bbdfdb80e
Author: Tim Allison <[email protected]>
AuthorDate: Wed Jun 24 15:23:19 2026 -0400
TIKA-4759 - use a markdown writer (#2900)
---
.../src/main/appended-resources/META-INF/LICENSE | 29 +
.../test/java/org/apache/tika/bundle/BundleIT.java | 9 +-
tika-bundles/tika-bundle-standard/test-bundles.xml | 3 +
tika-core/pom.xml | 14 +
.../apache/tika/sax/ToMarkdownContentHandler.java | 708 +++++++++++----------
.../tika/sax/ToMarkdownContentHandlerTest.java | 395 +++++++++++-
tika-parent/pom.xml | 16 +
tika-parsers/tika-parsers-ml/tika-vlm/pom.xml | 7 -
8 files changed, 801 insertions(+), 380 deletions(-)
diff --git a/tika-app/src/main/appended-resources/META-INF/LICENSE
b/tika-app/src/main/appended-resources/META-INF/LICENSE
index 43dae22216..ac394dcdaa 100644
--- a/tika-app/src/main/appended-resources/META-INF/LICENSE
+++ b/tika-app/src/main/appended-resources/META-INF/LICENSE
@@ -900,3 +900,32 @@ Sqlite (included in the "provided" org.xerial's
sqlite-jdbc)
Sqlite is in the Public Domain. For details
see: https://www.sqlite.org/copyright.html
+commonmark-java libraries (commonmark, commonmark-ext-gfm-tables, and
+commonmark-ext-gfm-strikethrough)
+
+ The BSD 2-Clause License
+
+ Copyright (c) 2015, Atlassian Pty Ltd
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions are met:
+
+ * Redistributions of source code must retain the above copyright notice,
this
+ list of conditions and the following disclaimer.
+
+ * Redistributions in binary form must reproduce the above copyright notice,
+ this list of conditions and the following disclaimer in the documentation
+ and/or other materials provided with the distribution.
+
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
ARE
+ DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
LIABLE
+ FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
LIABILITY,
+ OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
USE
+ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
diff --git
a/tika-bundles/tika-bundle-standard/src/test/java/org/apache/tika/bundle/BundleIT.java
b/tika-bundles/tika-bundle-standard/src/test/java/org/apache/tika/bundle/BundleIT.java
index 1a2d4fdd0d..76123b9b24 100644
---
a/tika-bundles/tika-bundle-standard/src/test/java/org/apache/tika/bundle/BundleIT.java
+++
b/tika-bundles/tika-bundle-standard/src/test/java/org/apache/tika/bundle/BundleIT.java
@@ -85,12 +85,19 @@ public class BundleIT {
// Install all bundles first, then start.
// tika-core requires osgi.serviceloader capabilities that are
// provided by tika-bundle-standard, so both must be installed
- // before either can resolve.
+ // before either can resolve. tika-core also imports the org.commonmark
+ // packages (Markdown serialization), so those bundles must be present
too.
Bundle commonsIo = install("commons-io.jar");
+ Bundle commonmark = install("commonmark.jar");
+ Bundle commonmarkTables = install("commonmark-ext-gfm-tables.jar");
+ Bundle commonmarkStrikethrough =
install("commonmark-ext-gfm-strikethrough.jar");
Bundle tikaCore = install("tika-core.jar");
Bundle tikaBundle = install("tika-bundle-standard.jar");
commonsIo.start();
+ commonmark.start();
+ commonmarkTables.start();
+ commonmarkStrikethrough.start();
tikaCore.start();
tikaBundle.start();
}
diff --git a/tika-bundles/tika-bundle-standard/test-bundles.xml
b/tika-bundles/tika-bundle-standard/test-bundles.xml
index 4da4310920..821846ec9c 100644
--- a/tika-bundles/tika-bundle-standard/test-bundles.xml
+++ b/tika-bundles/tika-bundle-standard/test-bundles.xml
@@ -30,6 +30,9 @@
<include>org.apache.tika:tika-core</include>
<include>org.apache.tika:tika-bundle-standard</include>
<include>commons-io:commons-io</include>
+ <include>org.commonmark:commonmark</include>
+ <include>org.commonmark:commonmark-ext-gfm-tables</include>
+ <include>org.commonmark:commonmark-ext-gfm-strikethrough</include>
</includes>
</dependencySet>
<dependencySet>
diff --git a/tika-core/pom.xml b/tika-core/pom.xml
index dcd0b780f2..030ad95c01 100644
--- a/tika-core/pom.xml
+++ b/tika-core/pom.xml
@@ -44,6 +44,20 @@
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
</dependency>
+ <!-- Markdown serialization (ToMarkdownContentHandler). BSD 2-Clause, ASF
Category A,
+ zero transitive dependencies. -->
+ <dependency>
+ <groupId>org.commonmark</groupId>
+ <artifactId>commonmark</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.commonmark</groupId>
+ <artifactId>commonmark-ext-gfm-tables</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.commonmark</groupId>
+ <artifactId>commonmark-ext-gfm-strikethrough</artifactId>
+ </dependency>
<!-- Optional OSGi dependencies, used only when running within OSGi -->
<dependency>
diff --git
a/tika-core/src/main/java/org/apache/tika/sax/ToMarkdownContentHandler.java
b/tika-core/src/main/java/org/apache/tika/sax/ToMarkdownContentHandler.java
index 13a54dcbd6..4228955394 100644
--- a/tika-core/src/main/java/org/apache/tika/sax/ToMarkdownContentHandler.java
+++ b/tika-core/src/main/java/org/apache/tika/sax/ToMarkdownContentHandler.java
@@ -23,69 +23,105 @@ import java.io.StringWriter;
import java.io.UnsupportedEncodingException;
import java.io.Writer;
import java.util.ArrayDeque;
-import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Deque;
import java.util.List;
import java.util.Locale;
+import org.commonmark.Extension;
+import org.commonmark.ext.gfm.strikethrough.Strikethrough;
+import org.commonmark.ext.gfm.strikethrough.StrikethroughExtension;
+import org.commonmark.ext.gfm.tables.TableBlock;
+import org.commonmark.ext.gfm.tables.TableBody;
+import org.commonmark.ext.gfm.tables.TableCell;
+import org.commonmark.ext.gfm.tables.TableHead;
+import org.commonmark.ext.gfm.tables.TableRow;
+import org.commonmark.ext.gfm.tables.TablesExtension;
+import org.commonmark.node.BlockQuote;
+import org.commonmark.node.BulletList;
+import org.commonmark.node.Code;
+import org.commonmark.node.Document;
+import org.commonmark.node.Emphasis;
+import org.commonmark.node.FencedCodeBlock;
+import org.commonmark.node.HardLineBreak;
+import org.commonmark.node.Heading;
+import org.commonmark.node.Image;
+import org.commonmark.node.Link;
+import org.commonmark.node.ListBlock;
+import org.commonmark.node.ListItem;
+import org.commonmark.node.Node;
+import org.commonmark.node.OrderedList;
+import org.commonmark.node.Paragraph;
+import org.commonmark.node.StrongEmphasis;
+import org.commonmark.node.Text;
+import org.commonmark.node.ThematicBreak;
+import org.commonmark.renderer.markdown.MarkdownRenderer;
import org.xml.sax.Attributes;
import org.xml.sax.SAXException;
import org.xml.sax.helpers.DefaultHandler;
/**
* SAX event handler that writes content as Markdown.
- * Supports headings, paragraphs, bold, italic, links, images, lists (ordered
- * and unordered, including nested), tables (GFM pipe tables), code blocks,
- * inline code, blockquotes, horizontal rules, and definition lists.
+ * Supports headings, paragraphs, bold, italic, strikethrough, links, images,
+ * lists (ordered and unordered, including nested), tables (GFM pipe tables),
+ * code blocks, inline code, blockquotes, horizontal rules, and definition
+ * lists.
+ * <p>
+ * The handler builds a <a
href="https://github.com/commonmark/commonmark-java";>
+ * commonmark-java</a> document model from the SAX event stream and renders it
+ * with commonmark's {@code MarkdownRenderer}. Document text is added to the
+ * model as raw literals, so escaping of Markdown metacharacters — including
+ * characters that would otherwise break out of a link, image, or table cell —
+ * is performed in one place by the renderer rather than at each emit site.
+ * <p>
+ * The handler tolerates malformed input (unbalanced or misnested tags): block
+ * elements are always attached at a structurally valid point so a well-formed
+ * document model is rendered regardless of the event stream.
* <p>
* Content within <script> and <style> tags is ignored.
- * </p>
*
* @since Apache Tika 3.2
*/
public class ToMarkdownContentHandler extends DefaultHandler {
- private static final String STYLE = "STYLE";
- private static final String SCRIPT = "SCRIPT";
+ private static final List<Extension> EXTENSIONS = Arrays.asList(
+ TablesExtension.create(), StrikethroughExtension.create());
private final Writer writer;
+ private final MarkdownRenderer renderer =
+ MarkdownRenderer.builder().extensions(EXTENSIONS).build();
- private final Deque<String> elementStack = new ArrayDeque<>();
- private final Deque<ListState> listStack = new ArrayDeque<>();
-
- // Link buffering
- private StringBuilder linkText;
- private String linkHref;
+ private final Document document = new Document();
+ private final Deque<Node> stack = new ArrayDeque<>();
- // Table buffering (only the outermost table is rendered; nested tables
are ignored)
- private int tableDepth = 0;
- private List<List<String>> tableRows;
- private List<String> currentRow;
- private StringBuilder currentCell;
+ // An implicit Paragraph opened to hold inline content that appeared in a
+ // block container (e.g. text directly under <body>/<li>/<blockquote>).
+ private Paragraph implicitParagraph;
- // Blockquote
- private int blockquoteDepth = 0;
+ // Buffer for inline <code> literal (commonmark Code holds a literal, not
children).
+ private Code inlineCode;
+ private StringBuilder inlineCodeText;
- // Code
- private boolean inPreBlock = false;
- private boolean inInlineCode = false;
+ // Buffer for <pre> fenced code block literal.
+ private FencedCodeBlock codeBlock;
+ private StringBuilder codeBlockText;
- // Script/style suppression
- private int scriptDepth = 0;
- private int styleDepth = 0;
+ // Table state (outermost table only).
+ private int tableDepth;
+ private TableBlock tableBlock;
+ private TableBody tableBody;
+ private int rowIndex;
+ private boolean inHeaderRow;
- // Spacing
- private boolean needsBlockSeparator = false;
- private boolean atLineStart = true;
+ // <script>/<style> content is dropped.
+ private int suppressDepth;
- // Track if we've written any content at all
- private boolean hasContent = false;
-
- // Track if meaningful (non-whitespace) content was written since last
block separator
- private boolean hasContentSinceLastSeparator = false;
+ // True once endDocument has rendered the full document to the writer.
+ private boolean finished;
public ToMarkdownContentHandler(Writer writer) {
this.writer = writer;
+ this.stack.push(document);
}
public ToMarkdownContentHandler(OutputStream stream, String encoding)
@@ -102,137 +138,132 @@ public class ToMarkdownContentHandler extends
DefaultHandler {
throws SAXException {
String name = localName(localName, qName);
- // Track script/style depth
- if (name.equals("script")) {
- scriptDepth++;
- elementStack.push(name);
- return;
- }
- if (name.equals("style")) {
- styleDepth++;
- elementStack.push(name);
+ if (name.equals("script") || name.equals("style")) {
+ suppressDepth++;
return;
}
-
- if (scriptDepth > 0 || styleDepth > 0) {
- elementStack.push(name);
+ if (suppressDepth > 0) {
return;
}
- elementStack.push(name);
-
switch (name) {
+ // --- block-level ---
case "h1":
case "h2":
case "h3":
case "h4":
case "h5":
case "h6":
- emitBlockSeparator();
- int level = name.charAt(1) - '0';
- write(repeatChar('#', level) + " ");
+ Heading heading = new Heading();
+ heading.setLevel(name.charAt(1) - '0');
+ openBlock(heading);
break;
case "p":
- emitBlockSeparator();
+ openBlock(new Paragraph());
break;
- case "b":
- case "strong":
- write("**");
- break;
- case "i":
- case "em":
- write("*");
- break;
- case "a":
- linkHref = atts.getValue("href");
- linkText = new StringBuilder();
- break;
- case "img":
- String alt = atts.getValue("alt");
- String src = atts.getValue("src");
- write("![" + escapeMarkdown(alt != null ? alt : "") + "](" +
- formatLinkDestination(src) + ")");
+ case "blockquote":
+ openBlock(new BlockQuote());
break;
case "ul":
+ openBlock(newBulletList());
+ break;
case "ol":
- if (!listStack.isEmpty()) {
- // nested list — no extra block separator
- } else {
- emitBlockSeparator();
- }
- listStack.push(new ListState(name.equals("ol"),
listStack.size()));
+ OrderedList ol = new OrderedList();
+ ol.setMarkerStartNumber(1);
+ ol.setMarkerDelimiter(".");
+ ol.setTight(true);
+ openBlock(ol);
break;
case "li":
- if (!listStack.isEmpty()) {
- ListState state = listStack.peek();
- String indent = repeatChar(' ', state.depth * 4);
- if (state.ordered) {
- state.counter++;
- write(indent + state.counter + ". ");
- } else {
- write(indent + "- ");
- }
- }
- break;
- case "blockquote":
- emitBlockSeparator();
- blockquoteDepth++;
+ openListItem();
break;
case "pre":
- emitBlockSeparator();
- inPreBlock = true;
- write("```\n");
- break;
- case "code":
- if (!inPreBlock) {
- inInlineCode = true;
- write("`");
- }
- break;
- case "br":
- write("\n");
- atLineStart = true;
+ popToBlockHost();
+ codeBlock = new FencedCodeBlock();
+ codeBlock.setFenceCharacter("`");
+ codeBlock.setOpeningFenceLength(3);
+ codeBlock.setLiteral("");
+ stack.peek().appendChild(codeBlock);
+ codeBlockText = new StringBuilder();
break;
case "hr":
- emitBlockSeparator();
- write("---");
- needsBlockSeparator = true;
- hasContent = true;
+ ThematicBreak hr = new ThematicBreak();
+ hr.setLiteral("---");
+ addBlockLeaf(hr);
+ break;
+ case "div":
+ // Block boundary: close any open paragraph so adjacent divs
separate.
+ popToBlockHost();
break;
case "table":
- tableDepth++;
- if (tableDepth == 1) {
- emitBlockSeparator();
- tableRows = new ArrayList<>();
- }
+ startTable();
break;
case "tr":
- if (tableDepth == 1 && tableRows != null) {
- currentRow = new ArrayList<>();
- }
+ startRow();
break;
case "th":
- if (tableDepth == 1 && currentRow != null) {
- currentCell = new StringBuilder();
- }
+ startCell(true);
break;
case "td":
- if (tableDepth == 1 && currentRow != null) {
- currentCell = new StringBuilder();
- }
+ startCell(false);
break;
case "dt":
- emitBlockSeparator();
- write("**");
+ Paragraph term = new Paragraph();
+ openBlock(term);
+ StrongEmphasis bold = new StrongEmphasis();
+ term.appendChild(bold);
+ stack.push(bold);
break;
case "dd":
- write("\n: ");
+ Paragraph def = new Paragraph();
+ openBlock(def);
+ def.appendChild(new Text(": "));
break;
- case "div":
- emitBlockSeparator();
+
+ // --- inline-level ---
+ case "b":
+ case "strong":
+ openInline(new StrongEmphasis());
+ break;
+ case "i":
+ case "em":
+ openInline(new Emphasis());
+ break;
+ case "s":
+ case "strike":
+ case "del":
+ openInline(new Strikethrough("~~"));
break;
+ case "a":
+ String href = atts.getValue("href");
+ openInline(new Link(href != null ? href : "", null));
+ break;
+ case "code":
+ if (codeBlockText == null) {
+ ensureInlineContainer();
+ inlineCode = new Code("");
+ stack.peek().appendChild(inlineCode);
+ inlineCodeText = new StringBuilder();
+ }
+ break;
+ case "br":
+ ensureInlineContainer();
+ stack.peek().appendChild(new HardLineBreak());
+ break;
+ case "img":
+ ensureInlineContainer();
+ Image img = new Image(value(atts, "src"), null);
+ String alt = atts.getValue("alt");
+ if (alt != null && !alt.isEmpty()) {
+ img.appendChild(new Text(alt));
+ }
+ stack.peek().appendChild(img);
+ break;
+
default:
- // Ignore structural elements like html, head, body, title,
meta
+ // html, head, body, title, meta, div, span, dl, thead,
tbody...
+ // structurally transparent; their inline children flow to the
+ // nearest real container.
break;
}
}
@@ -241,21 +272,11 @@ public class ToMarkdownContentHandler extends
DefaultHandler {
public void endElement(String uri, String localName, String qName) throws
SAXException {
String name = localName(localName, qName);
- if (!elementStack.isEmpty()) {
- elementStack.pop();
- }
-
- // Track script/style depth
- if (name.equals("script")) {
- scriptDepth--;
+ if (name.equals("script") || name.equals("style")) {
+ suppressDepth = Math.max(0, suppressDepth - 1);
return;
}
- if (name.equals("style")) {
- styleDepth--;
- return;
- }
-
- if (scriptDepth > 0 || styleDepth > 0) {
+ if (suppressDepth > 0) {
return;
}
@@ -266,98 +287,61 @@ public class ToMarkdownContentHandler extends
DefaultHandler {
case "h4":
case "h5":
case "h6":
- needsBlockSeparator = true;
- hasContent = true;
- break;
case "p":
- needsBlockSeparator = true;
- hasContent = true;
- break;
+ case "blockquote":
+ case "ul":
+ case "ol":
+ case "li":
case "b":
case "strong":
- write("**");
- break;
case "i":
case "em":
- write("*");
- break;
+ case "s":
+ case "strike":
+ case "del":
case "a":
- if (linkText != null) {
- write("[" + escapeMarkdown(linkText.toString()) + "](" +
- formatLinkDestination(linkHref) + ")");
- linkText = null;
- linkHref = null;
- }
+ closeImplicitParagraph();
+ pop();
break;
- case "ul":
- case "ol":
- if (!listStack.isEmpty()) {
- listStack.pop();
- }
- if (listStack.isEmpty()) {
- needsBlockSeparator = true;
- hasContent = true;
- }
- break;
- case "li":
- write("\n");
- atLineStart = true;
+ case "dt":
+ pop(); // StrongEmphasis
+ pop(); // Paragraph
break;
- case "blockquote":
- blockquoteDepth--;
- needsBlockSeparator = true;
- hasContent = true;
+ case "dd":
+ pop();
break;
case "pre":
- if (!endsWithNewline()) {
- write("\n");
+ if (codeBlock != null) {
+
codeBlock.setLiteral(withTrailingNewline(codeBlockText.toString()));
+ codeBlock = null;
+ codeBlockText = null;
}
- write("```");
- inPreBlock = false;
- needsBlockSeparator = true;
- hasContent = true;
break;
case "code":
- if (!inPreBlock) {
- inInlineCode = false;
- write("`");
+ if (inlineCode != null) {
+ inlineCode.setLiteral(inlineCodeText.toString());
+ inlineCode = null;
+ inlineCodeText = null;
}
break;
+ case "div":
+ closeImplicitParagraph();
+ break;
case "table":
- if (tableDepth == 1) {
- emitTable();
- tableRows = null;
- currentRow = null;
- currentCell = null;
- needsBlockSeparator = true;
- hasContent = true;
- }
- tableDepth = Math.max(0, tableDepth - 1);
+ endTable();
break;
case "tr":
- if (tableDepth == 1 && tableRows != null && currentRow !=
null) {
- tableRows.add(currentRow);
- currentRow = null;
+ if (tableDepth == 1) {
+ pop();
+ rowIndex++;
}
break;
case "th":
case "td":
- if (tableDepth == 1 && currentRow != null && currentCell !=
null) {
- currentRow.add(escapeTableCell(currentCell.toString()));
- currentCell = null;
+ if (tableDepth == 1) {
+ pop();
}
break;
- case "dt":
- write("**");
- break;
- case "dd":
- needsBlockSeparator = true;
- hasContent = true;
- break;
- case "div":
- needsBlockSeparator = true;
- hasContent = true;
- break;
default:
break;
}
@@ -365,58 +349,34 @@ public class ToMarkdownContentHandler extends
DefaultHandler {
@Override
public void characters(char[] ch, int start, int length) throws
SAXException {
- if (scriptDepth > 0 || styleDepth > 0) {
+ if (suppressDepth > 0) {
return;
}
-
- // Buffer into link text
- if (linkText != null) {
- linkText.append(ch, start, length);
+ if (codeBlockText != null) {
+ codeBlockText.append(ch, start, length);
return;
}
-
- // Buffer into table cell
- if (currentCell != null) {
- currentCell.append(ch, start, length);
+ if (inlineCodeText != null) {
+ inlineCodeText.append(ch, start, length);
return;
}
String text = new String(ch, start, length);
-
- // In pre blocks, write raw (no escaping)
- if (inPreBlock) {
- write(text);
- return;
- }
-
- // In inline code, write raw (no escaping)
- if (inInlineCode) {
- write(text);
- return;
- }
-
- // Skip whitespace-only text at line start; preserve inline spaces
if (text.trim().isEmpty()) {
- if (!atLineStart) {
- write(" ");
+ // Inter-element whitespace: drop between blocks, collapse within
inline runs.
+ if (!isBlockContainer(stack.peek())) {
+ stack.peek().appendChild(new Text(" "));
}
return;
}
- // Escape markdown special characters in normal text
- text = escapeMarkdown(text);
-
- // Add blockquote prefix if needed at line start
- if (blockquoteDepth > 0 && atLineStart && !text.isEmpty()) {
- write(repeatChar('>', blockquoteDepth) + " ");
- atLineStart = false;
- }
-
- if (!text.isEmpty()) {
- write(text);
- hasContent = true;
- hasContentSinceLastSeparator = true;
+ if (isStructuralOnly(stack.peek())) {
+ // Stray text inside a list/table wrapper — not a valid inline
position; drop.
+ return;
}
+ ensureInlineContainer();
+ // Newlines inside prose would otherwise survive as literal line
breaks.
+ stack.peek().appendChild(new Text(collapseLineBreaks(text)));
}
@Override
@@ -427,132 +387,193 @@ public class ToMarkdownContentHandler extends
DefaultHandler {
@Override
public void endDocument() throws SAXException {
try {
+ renderer.render(document, writer);
writer.flush();
+ finished = true;
} catch (IOException e) {
- throw new SAXException("Error flushing character output", e);
+ throw new SAXException("Error writing markdown", e);
}
}
@Override
public String toString() {
- return writer.toString();
+ if (finished) {
+ return writer.toString();
+ }
+ // Parsing did not complete (e.g. the parser threw mid-document).
Render the
+ // content accumulated so far so partial content is not lost on
failure — the
+ // streaming writer this replaced exposed partial content the same way.
+ return renderer.render(document);
}
- private void write(String s) throws SAXException {
- try {
- writer.write(s);
- if (!s.isEmpty()) {
- atLineStart = s.charAt(s.length() - 1) == '\n';
- if (!s.trim().isEmpty()) {
- hasContentSinceLastSeparator = true;
- }
+ // --- tree construction helpers ---
+
+ /**
+ * Append a block-level node at a structurally valid point and descend into
+ * it. commonmark requires a block's parent to be a block, so any inline
(or
+ * other non-hosting) nodes left open are closed first.
+ */
+ private void openBlock(Node block) {
+ popToBlockHost();
+ stack.peek().appendChild(block);
+ stack.push(block);
+ }
+
+ /** Append a block-level leaf (hr, code block, table) without descending.
*/
+ private void addBlockLeaf(Node block) {
+ popToBlockHost();
+ stack.peek().appendChild(block);
+ }
+
+ /**
+ * A list item is only valid inside a list. If the current point isn't a
+ * list (a stray {@code <li>} or misnested markup), recover by opening an
+ * implicit bullet list so the model stays renderable.
+ */
+ private void openListItem() {
+ if (!(stack.peek() instanceof ListBlock)) {
+ popToBlockHost();
+ BulletList implicit = newBulletList();
+ stack.peek().appendChild(implicit);
+ stack.push(implicit);
+ }
+ ListItem li = new ListItem();
+ stack.peek().appendChild(li);
+ stack.push(li);
+ }
+
+ private void openInline(Node node) {
+ ensureInlineContainer();
+ stack.peek().appendChild(node);
+ stack.push(node);
+ }
+
+ private void pop() {
+ if (stack.size() > 1) {
+ Node popped = stack.pop();
+ if (popped == implicitParagraph) {
+ implicitParagraph = null;
}
- } catch (IOException e) {
- throw new SAXException("Error writing: " + s, e);
}
}
- private void emitBlockSeparator() throws SAXException {
- if (needsBlockSeparator && hasContent && hasContentSinceLastSeparator)
{
- write("\n\n");
- needsBlockSeparator = false;
- atLineStart = true;
- hasContentSinceLastSeparator = false;
- } else {
- needsBlockSeparator = false;
+ /** Pop open nodes until the top can host block children
(Document/blockquote/list item). */
+ private void popToBlockHost() {
+ while (stack.size() > 1 && !canHostBlocks(stack.peek())) {
+ Node popped = stack.pop();
+ if (popped == implicitParagraph) {
+ implicitParagraph = null;
+ }
}
}
- private void emitTable() throws SAXException {
- if (tableRows == null || tableRows.isEmpty()) {
- return;
+ /**
+ * If the current insertion point is a block container that cannot hold
+ * inline content directly, open an implicit {@link Paragraph} to hold it.
+ */
+ private void ensureInlineContainer() {
+ Node top = stack.peek();
+ if (canHostBlocks(top)) {
+ Paragraph p = new Paragraph();
+ top.appendChild(p);
+ stack.push(p);
+ implicitParagraph = p;
}
+ }
- // Determine column count
- int cols = 0;
- for (List<String> row : tableRows) {
- cols = Math.max(cols, row.size());
+ private void closeImplicitParagraph() {
+ if (implicitParagraph != null && stack.peek() == implicitParagraph) {
+ stack.pop();
+ implicitParagraph = null;
}
+ }
- // Emit rows
- for (int r = 0; r < tableRows.size(); r++) {
- List<String> row = tableRows.get(r);
- StringBuilder sb = new StringBuilder("|");
- for (int c = 0; c < cols; c++) {
- String cell = c < row.size() ? row.get(c) : "";
- sb.append(" ").append(cell).append(" |");
- }
- write(sb.toString());
- write("\n");
-
- // Insert separator after first row
- if (r == 0) {
- StringBuilder sep = new StringBuilder("|");
- for (int c = 0; c < cols; c++) {
- sep.append(" --- |");
- }
- write(sep.toString());
- write("\n");
- }
+ private void startTable() {
+ tableDepth++;
+ if (tableDepth == 1) {
+ popToBlockHost();
+ tableBlock = new TableBlock();
+ tableBody = null;
+ rowIndex = 0;
+ stack.peek().appendChild(tableBlock);
}
}
- private boolean endsWithNewline() {
- String s = writer.toString();
- return !s.isEmpty() && s.charAt(s.length() - 1) == '\n';
+ private void endTable() {
+ if (tableDepth == 1) {
+ tableBlock = null;
+ tableBody = null;
+ }
+ tableDepth = Math.max(0, tableDepth - 1);
}
- private static String escapeMarkdown(String text) {
- StringBuilder sb = new StringBuilder(text.length());
- for (int i = 0; i < text.length(); i++) {
- char c = text.charAt(i);
- switch (c) {
- case '\\':
- case '`':
- case '*':
- case '_':
- case '[':
- case ']':
- case '#':
- case '|':
- sb.append('\\').append(c);
- break;
- default:
- sb.append(c);
- break;
+ private void startRow() {
+ if (tableDepth != 1) {
+ return;
+ }
+ TableRow row = new TableRow();
+ if (rowIndex == 0) {
+ TableHead head = new TableHead();
+ tableBlock.appendChild(head);
+ head.appendChild(row);
+ inHeaderRow = true;
+ } else {
+ if (tableBody == null) {
+ tableBody = new TableBody();
+ tableBlock.appendChild(tableBody);
}
+ tableBody.appendChild(row);
+ inHeaderRow = false;
}
- return sb.toString();
+ stack.push(row);
}
- private static String escapeTableCell(String text) {
- // a newline ends a GFM table row and a pipe ends a cell; fold the
former and
- // escape the latter (with the other markers) so cell content can't
break out
- // of its column or row.
- return escapeMarkdown(text.replaceAll("[\\r\\n]+", " ").trim());
+ private void startCell(boolean header) {
+ if (tableDepth != 1 || !(stack.peek() instanceof TableRow)) {
+ return;
+ }
+ TableCell cell = new TableCell();
+ cell.setHeader(header || inHeaderRow);
+ stack.peek().appendChild(cell);
+ stack.push(cell);
}
- private static String formatLinkDestination(String url) {
- if (url == null || url.isEmpty()) {
- return "";
- }
- // angle brackets and line breaks can't appear in any markdown
destination
- String cleaned = url.replaceAll("[\\u0000-\\u001f<>]", "");
- // a space or paren would otherwise close the bare (url) form early,
so wrap
- // those in <>, where spaces and parens are allowed
- if (cleaned.indexOf(' ') >= 0 || cleaned.indexOf('(') >= 0
- || cleaned.indexOf(')') >= 0) {
- return "<" + cleaned + ">";
- }
- return cleaned;
+ private static BulletList newBulletList() {
+ BulletList list = new BulletList();
+ list.setMarker("-");
+ list.setTight(true);
+ return list;
+ }
+
+ private static boolean canHostBlocks(Node n) {
+ return n instanceof Document || n instanceof BlockQuote || n
instanceof ListItem;
+ }
+
+ private static boolean isBlockContainer(Node n) {
+ return n instanceof Document || n instanceof BlockQuote || n
instanceof ListItem
+ || n instanceof ListBlock || n instanceof TableBlock || n
instanceof TableHead
+ || n instanceof TableBody || n instanceof TableRow;
+ }
+
+ private static boolean isStructuralOnly(Node n) {
+ return n instanceof ListBlock || n instanceof TableBlock || n
instanceof TableHead
+ || n instanceof TableBody || n instanceof TableRow;
+ }
+
+ private static String collapseLineBreaks(String s) {
+ return s.replace('\r', ' ').replace('\n', ' ');
}
- private static String repeatChar(char c, int count) {
- StringBuilder sb = new StringBuilder(count);
- for (int i = 0; i < count; i++) {
- sb.append(c);
+ private static String withTrailingNewline(String s) {
+ if (s.isEmpty() || s.charAt(s.length() - 1) == '\n') {
+ return s;
}
- return sb.toString();
+ return s + "\n";
+ }
+
+ private static String value(Attributes atts, String name) {
+ String v = atts.getValue(name);
+ return v != null ? v : "";
}
private static String localName(String localName, String qName) {
@@ -560,23 +581,10 @@ public class ToMarkdownContentHandler extends
DefaultHandler {
return localName.toLowerCase(Locale.ROOT);
}
if (qName != null) {
- // Strip namespace prefix
int colon = qName.indexOf(':');
String name = colon >= 0 ? qName.substring(colon + 1) : qName;
return name.toLowerCase(Locale.ROOT);
}
return "";
}
-
- private static class ListState {
- final boolean ordered;
- final int depth;
- int counter;
-
- ListState(boolean ordered, int depth) {
- this.ordered = ordered;
- this.depth = depth;
- this.counter = 0;
- }
- }
}
diff --git
a/tika-core/src/test/java/org/apache/tika/sax/ToMarkdownContentHandlerTest.java
b/tika-core/src/test/java/org/apache/tika/sax/ToMarkdownContentHandlerTest.java
index c4bcf6323d..c0437aff5c 100644
---
a/tika-core/src/test/java/org/apache/tika/sax/ToMarkdownContentHandlerTest.java
+++
b/tika-core/src/test/java/org/apache/tika/sax/ToMarkdownContentHandlerTest.java
@@ -21,6 +21,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
+import java.util.Locale;
import java.util.Random;
import org.junit.jupiter.api.RepeatedTest;
@@ -62,6 +63,53 @@ public class ToMarkdownContentHandlerTest {
handler.characters(ch, 0, ch.length);
}
+ /** A replayable sequence of SAX events, for the behaviour-lock tests
below. */
+ private interface Events {
+ void emit(ContentHandler h) throws Exception;
+ }
+
+ /**
+ * Render the given events and assert the handler does not throw,
returning the
+ * Markdown. The Markdown is now produced by building a commonmark AST,
which is
+ * stricter than the string writer it replaced, so these tests both
document the
+ * handler's tolerance of malformed/illegal input and fail loudly if a
future
+ * commonmark upgrade starts throwing on input we currently render.
+ */
+ private static String renderNoThrow(String label, Events events) {
+ ToMarkdownContentHandler handler = new ToMarkdownContentHandler();
+ return assertDoesNotThrow(() -> {
+ handler.startDocument();
+ events.emit(handler);
+ handler.endDocument();
+ return handler.toString();
+ }, label);
+ }
+
+ /**
+ * Emits a compact event script: a bare tag opens an element ({@code
"ul"}), a
+ * {@code "/"}-prefixed tag closes one ({@code "/ul"}), and a {@code
"#"}-prefixed
+ * token is character data ({@code "#text"}). {@code "a"} opens an anchor
and
+ * {@code "img"} an image, each with placeholder attributes.
+ */
+ private static void script(ContentHandler h, String... ops) throws
Exception {
+ for (String op : ops) {
+ if (op.startsWith("/")) {
+ endElement(h, op.substring(1));
+ } else if (op.startsWith("#")) {
+ chars(h, op.substring(1));
+ } else if (op.equals("a")) {
+ startElement(h, "a", "href", "http://e";);
+ } else if (op.equals("img")) {
+ AttributesImpl atts = new AttributesImpl();
+ atts.addAttribute("", "src", "src", "CDATA", "i.png");
+ atts.addAttribute("", "alt", "alt", "CDATA", "alt");
+ startElement(h, "img", atts);
+ } else {
+ startElement(h, op);
+ }
+ }
+ }
+
@Test
public void testHeadings() throws Exception {
ToMarkdownContentHandler handler = new ToMarkdownContentHandler();
@@ -314,8 +362,8 @@ public class ToMarkdownContentHandlerTest {
String result = handler.toString();
assertTrue(result.contains("- Fruit"));
- assertTrue(result.contains(" - Apple"));
- assertTrue(result.contains(" - Banana"));
+ assertTrue(result.contains(" - Apple"));
+ assertTrue(result.contains(" - Banana"));
assertTrue(result.contains("- Vegetable"));
}
@@ -351,9 +399,9 @@ public class ToMarkdownContentHandlerTest {
handler.endDocument();
String result = handler.toString();
- assertTrue(result.contains("| Name | Age |"));
- assertTrue(result.contains("| --- | --- |"));
- assertTrue(result.contains("| Alice | 30 |"));
+ assertTrue(result.contains("|Name|Age|"));
+ assertTrue(result.contains("|---|---|"));
+ assertTrue(result.contains("|Alice|30|"));
}
@Test
@@ -445,7 +493,8 @@ public class ToMarkdownContentHandlerTest {
handler.endDocument();
- assertTrue(handler.toString().contains("Line one\nLine two"));
+ // GFM hard line break: two trailing spaces before the newline
+ assertTrue(handler.toString().contains("Line one \nLine two"));
}
@Test
@@ -544,7 +593,9 @@ public class ToMarkdownContentHandlerTest {
assertTrue(result.contains("\\_"));
assertTrue(result.contains("\\["));
assertTrue(result.contains("\\]"));
- assertTrue(result.contains("\\#"));
+ // '#' only needs escaping at line start, so it is left bare mid-line
+ assertTrue(result.contains("#"));
+ assertFalse(result.contains("\\#"));
assertTrue(result.contains("\\|"));
assertTrue(result.contains("\\\\"));
assertTrue(result.contains("\\`"));
@@ -776,7 +827,7 @@ public class ToMarkdownContentHandlerTest {
String result = handler.toString();
// a pipe inside a cell must be escaped so it does not inject an extra
column
- assertTrue(result.contains("| a\\|b | c |"), result);
+ assertTrue(result.contains("|a\\|b|c|"), result);
}
@Test
@@ -796,7 +847,7 @@ public class ToMarkdownContentHandlerTest {
String result = handler.toString();
// a newline inside a cell must not terminate the table row
- assertTrue(result.contains("| a b |"), result);
+ assertTrue(result.contains("|a b|"), result);
}
@Test
@@ -853,9 +904,9 @@ public class ToMarkdownContentHandlerTest {
handler.endDocument();
String result = handler.toString();
- assertTrue(result.contains("| A | B |"));
- assertTrue(result.contains("| --- | --- |"));
- assertTrue(result.contains("| C | D |"));
+ assertTrue(result.contains("|A|B|"));
+ assertTrue(result.contains("|---|---|"));
+ assertTrue(result.contains("|C|D|"));
}
@Test
@@ -901,12 +952,12 @@ public class ToMarkdownContentHandlerTest {
String result = handler.toString();
// Outer table should be rendered
- assertTrue(result.contains("| Outer1 | Outer2 |"));
- assertTrue(result.contains("| --- | --- |"));
+ assertTrue(result.contains("|Outer1|Outer2|"));
+ assertTrue(result.contains("|---|---|"));
// Inner cell text gets folded into the outer cell ("B" + "Inner" =
"BInner")
- assertTrue(result.contains("| A | BInner |"));
+ assertTrue(result.contains("|A|BInner|"));
// Inner table structure should not appear as a separate table
- assertFalse(result.contains("| Inner |"));
+ assertFalse(result.contains("|Inner|"));
}
private static final String[] ALL_ELEMENTS = {
@@ -929,7 +980,7 @@ public class ToMarkdownContentHandlerTest {
* runtime exceptions (e.g., EmptyStackException, NullPointerException,
* IndexOutOfBoundsException).
*/
- @RepeatedTest(20)
+ @RepeatedTest(100)
public void testRandomUnbalancedTags() throws Exception {
Random rng = new Random();
ToMarkdownContentHandler handler = new ToMarkdownContentHandler();
@@ -945,11 +996,11 @@ public class ToMarkdownContentHandlerTest {
case 0:
// start element (possibly with attributes)
if (elem.equals("a")) {
- startElement(handler, elem, "href",
"http://example.com";);
+ startElement(handler, elem, "href",
randomText(rng));
} else if (elem.equals("img")) {
AttributesImpl atts = new AttributesImpl();
- atts.addAttribute("", "src", "src", "CDATA",
"img.png");
- atts.addAttribute("", "alt", "alt", "CDATA", "alt
text");
+ atts.addAttribute("", "src", "src", "CDATA",
randomText(rng));
+ atts.addAttribute("", "alt", "alt", "CDATA",
randomText(rng));
startElement(handler, elem, atts);
} else {
startElement(handler, elem);
@@ -960,8 +1011,10 @@ public class ToMarkdownContentHandlerTest {
endElement(handler, elem);
break;
case 2:
- // characters
- chars(handler, "text_" + i);
+ // characters -- random content incl. control chars,
null bytes,
+ // unpaired surrogates and non-characters (NOT relying
on an
+ // upstream SafeContentHandler to have cleaned them)
+ chars(handler, randomText(rng));
break;
case 3:
// ignorable whitespace
@@ -1040,6 +1093,81 @@ public class ToMarkdownContentHandlerTest {
});
}
+ @Test
+ public void testMisnestedBlockInsideInlineAndCrossedClose() throws
Exception {
+ // Classic broken nesting a misbehaving parser can emit: a block opened
+ // inside an inline, with crossed end tags (<p><div></p></div>). The
AST
+ // builder must not throw (commonmark rejects a block under an inline)
and
+ // must still render the text.
+ ToMarkdownContentHandler handler = new ToMarkdownContentHandler();
+ String result = assertDoesNotThrow(() -> {
+ handler.startDocument();
+ startElement(handler, "p");
+ startElement(handler, "div");
+ chars(handler, "misnested text");
+ endElement(handler, "p");
+ endElement(handler, "div");
+ handler.endDocument();
+ return handler.toString();
+ });
+ assertContains("misnested text", result);
+ }
+
+ @Test
+ public void testInlineSpanningBlockBoundaries() throws Exception {
+ // <b>bold<p>para</b>more</p>: inline opened, a block opened inside
it, the
+ // inline closed across the block boundary, then more text. Must not
throw
+ // and must preserve all the text.
+ ToMarkdownContentHandler handler = new ToMarkdownContentHandler();
+ String result = assertDoesNotThrow(() -> {
+ handler.startDocument();
+ startElement(handler, "b");
+ chars(handler, "bold");
+ startElement(handler, "p");
+ chars(handler, "para");
+ endElement(handler, "b");
+ chars(handler, "more");
+ endElement(handler, "p");
+ handler.endDocument();
+ return handler.toString();
+ });
+ assertContains("bold", result);
+ assertContains("para", result);
+ assertContains("more", result);
+ }
+
+ @Test
+ public void testContentPreservedWhenParserThrowsBeforeEndDocument() throws
Exception {
+ // A parser that writes content and then throws never calls
endDocument().
+ // Because this handler builds an AST and renders at endDocument,
partial
+ // content would be lost if toString() relied solely on a completed
render.
+ // It must instead render what has been received so far -- matching the
+ // streaming writer this replaced. (Regression guard for the rmeta NPE
case.)
+ ToMarkdownContentHandler handler = new ToMarkdownContentHandler();
+ handler.startDocument();
+ startElement(handler, "p");
+ chars(handler, "some content");
+ endElement(handler, "p");
+ // deliberately NO endDocument() -- simulates a parser exception
mid-document
+ assertContains("some content", handler.toString());
+ }
+
+ @Test
+ public void testPartialContentWithStillOpenElementsNoEndDocument() throws
Exception {
+ // Same failure mode, but the exception lands while elements are still
open
+ // (no end tags, no endDocument). Content received so far must still
surface.
+ ToMarkdownContentHandler handler = new ToMarkdownContentHandler();
+ handler.startDocument();
+ startElement(handler, "h1");
+ chars(handler, "Heading");
+ startElement(handler, "p");
+ chars(handler, "body text");
+ // no end tags, no endDocument()
+ String result = handler.toString();
+ assertContains("Heading", result);
+ assertContains("body text", result);
+ }
+
/**
* Test deeply nested elements of the same type -- should not throw.
*/
@@ -1108,4 +1236,227 @@ public class ToMarkdownContentHandlerTest {
handler.endDocument();
});
}
+
+ // --- untrusted content cannot break out of inline/table structure ---
+
+ @Test
+ public void testLinkTextCannotBreakOut() throws Exception {
+ ToMarkdownContentHandler handler = new ToMarkdownContentHandler();
+ handler.startDocument();
+ startElement(handler, "p");
+ startElement(handler, "a", "href", "https://good.example";);
+ chars(handler, "x](https://evil.example)");
+ endElement(handler, "a");
+ endElement(handler, "p");
+ handler.endDocument();
+
+ String result = handler.toString();
+ // the ] in the link text is escaped, so it cannot close the link early
+ assertTrue(result.contains("x\\]"), result);
+ assertFalse(result.contains("x](https://evil.example)"), result);
+ assertTrue(result.contains("](https://good.example)"), result);
+ }
+
+ @Test
+ public void testLinkDestinationWithSpaceAndParenWrapped() throws Exception
{
+ ToMarkdownContentHandler handler = new ToMarkdownContentHandler();
+ handler.startDocument();
+ startElement(handler, "p");
+ startElement(handler, "a", "href", "https://evil.example) text");
+ chars(handler, "click");
+ endElement(handler, "a");
+ endElement(handler, "p");
+ handler.endDocument();
+
+ String result = handler.toString();
+ // a destination with spaces/parens is wrapped in <> so it cannot
terminate early
+ assertTrue(result.contains("](<https://evil.example) text>)"), result);
+ }
+
+ @Test
+ public void testTableCellPipeAndNewlineContained() throws Exception {
+ ToMarkdownContentHandler handler = new ToMarkdownContentHandler();
+ handler.startDocument();
+ startElement(handler, "table");
+ startElement(handler, "tr");
+ startElement(handler, "th");
+ chars(handler, "h1");
+ endElement(handler, "th");
+ startElement(handler, "th");
+ chars(handler, "h2");
+ endElement(handler, "th");
+ endElement(handler, "tr");
+ startElement(handler, "tr");
+ startElement(handler, "td");
+ chars(handler, "a|b\nc");
+ endElement(handler, "td");
+ startElement(handler, "td");
+ chars(handler, "d");
+ endElement(handler, "td");
+ endElement(handler, "tr");
+ endElement(handler, "table");
+ handler.endDocument();
+
+ String result = handler.toString();
+ // pipe escaped, newline folded: cell stays one cell, row stays two
columns
+ assertTrue(result.contains("|a\\|b c|d|"), result);
+ }
+
+ @Test
+ public void testImageAltAndSrcCannotBreakOut() throws Exception {
+ ToMarkdownContentHandler handler = new ToMarkdownContentHandler();
+ handler.startDocument();
+ startElement(handler, "p");
+ AttributesImpl atts = new AttributesImpl();
+ atts.addAttribute("", "alt", "alt", "CDATA", "a]b");
+ atts.addAttribute("", "src", "src", "CDATA", "https://evil.example)
x");
+ startElement(handler, "img", atts);
+ endElement(handler, "img");
+ endElement(handler, "p");
+ handler.endDocument();
+
+ String result = handler.toString();
+ assertTrue(result.contains("a\\]b"), result);
+ assertFalse(result.contains("](https://evil.example) x)"), result);
+ }
+
+ @Test
+ public void testBoldInsideLinkIsNested() throws Exception {
+ ToMarkdownContentHandler handler = new ToMarkdownContentHandler();
+ handler.startDocument();
+ startElement(handler, "p");
+ startElement(handler, "a", "href", "https://example.com";);
+ startElement(handler, "b");
+ chars(handler, "bold link");
+ endElement(handler, "b");
+ endElement(handler, "a");
+ endElement(handler, "p");
+ handler.endDocument();
+
+ String result = handler.toString();
+ // inline markup nests correctly inside the link text
+ assertTrue(result.contains("[**bold link**](https://example.com)"),
result);
+ }
+
+ @Test
+ public void testInlineContentInStructuralContainersDoesNotThrow() {
+ // A parser may emit inline content directly inside a list/table
container
+ // (no list item / cell). commonmark tolerates the resulting tree
today.
+ renderNoThrow("bold in ul", h -> script(h, "ul", "b", "#x", "/b",
"/ul"));
+ renderNoThrow("italic in ol", h -> script(h, "ol", "i", "#x", "/i",
"/ol"));
+ renderNoThrow("bold in tr without cell",
+ h -> script(h, "table", "tr", "b", "#x", "/b", "/tr",
"/table"));
+ renderNoThrow("inline directly in table",
+ h -> script(h, "table", "b", "#x", "/b", "/table"));
+ renderNoThrow("img in tr without cell",
+ h -> script(h, "table", "tr", "img", "/img", "/tr", "/table"));
+ renderNoThrow("br in ul", h -> script(h, "ul", "br", "/br", "/ul"));
+ renderNoThrow("link in ul", h -> script(h, "ul", "a", "#x", "/a",
"/ul"));
+ renderNoThrow("heading inside link", h -> script(h, "a", "h1", "#x",
"/h1", "/a"));
+ renderNoThrow("table inside paragraph",
+ h -> script(h, "p", "table", "tr", "td", "#x", "/td", "/tr",
"/table", "/p"));
+ renderNoThrow("list inside table",
+ h -> script(h, "table", "ul", "li", "#x", "/li", "/ul",
"/table"));
+ }
+
+ @Test
+ public void testTableAndListEdgeCasesDoNotThrow() {
+ renderNoThrow("empty table", h -> script(h, "table", "/table"));
+ renderNoThrow("table with only th", h -> script(h, "table", "th",
"#x", "/th", "/table"));
+ renderNoThrow("table with empty cells",
+ h -> script(h, "table", "tr", "td", "/td", "/tr", "/table"));
+ renderNoThrow("td outside table", h -> script(h, "td", "#x", "/td"));
+ renderNoThrow("tr outside table", h -> script(h, "tr", "td", "#x",
"/td", "/tr"));
+ renderNoThrow("li without a list", h -> script(h, "li", "#x", "/li"));
+ renderNoThrow("stray text between list items",
+ h -> script(h, "ol", "li", "#a", "/li", "#stray", "li", "#b",
"/li", "/ol"));
+ }
+
+ @Test
+ public void testDeeplyNestedElementsDoNotThrow() {
+ // Pathologically deep nesting: the renderer walks the tree
recursively.
+ renderNoThrow("blockquote x1000", h -> {
+ for (int i = 0; i < 1000; i++) {
+ startElement(h, "blockquote");
+ }
+ chars(h, "deep");
+ for (int i = 0; i < 1000; i++) {
+ endElement(h, "blockquote");
+ }
+ });
+ renderNoThrow("div x2000", h -> {
+ for (int i = 0; i < 2000; i++) {
+ startElement(h, "div");
+ }
+ chars(h, "deep");
+ for (int i = 0; i < 2000; i++) {
+ endElement(h, "div");
+ }
+ });
+ }
+
+ @RepeatedTest(200)
+ public void fuzzRandomCharacterContent() {
+ // The handler must not throw on arbitrary character content on its
own --
+ // control chars, null bytes, unpaired surrogates, non-characters. It
does NOT
+ // rely on SafeContentHandler having sanitized the stream: that runs
in the parse
+ // pipeline, but this handler is public API and may be used directly.
+ Random rng = new Random();
+ String text = randomText(rng);
+
+ assertDoesNotThrow(() -> {
+ ToMarkdownContentHandler handler = new ToMarkdownContentHandler();
+ handler.startDocument();
+ startElement(handler, "p");
+ chars(handler, text);
+ endElement(handler, "p");
+ startElement(handler, "table");
+ startElement(handler, "tr");
+ startElement(handler, "td");
+ chars(handler, text);
+ endElement(handler, "td");
+ endElement(handler, "tr");
+ endElement(handler, "table");
+ AttributesImpl atts = new AttributesImpl();
+ atts.addAttribute("", "src", "src", "CDATA", text);
+ atts.addAttribute("", "alt", "alt", "CDATA", text);
+ startElement(handler, "p");
+ startElement(handler, "a", "href", text);
+ chars(handler, text);
+ endElement(handler, "a");
+ startElement(handler, "img", atts);
+ endElement(handler, "img");
+ endElement(handler, "p");
+ handler.endDocument();
+ return handler.toString();
+ }, () -> "StringWriter path threw for input: " + describe(text));
+
+ assertDoesNotThrow(() -> {
+ java.io.ByteArrayOutputStream bos = new
java.io.ByteArrayOutputStream();
+ ToMarkdownContentHandler handler = new
ToMarkdownContentHandler(bos, "UTF-8");
+ handler.startDocument();
+ startElement(handler, "p");
+ chars(handler, text);
+ endElement(handler, "p");
+ handler.endDocument();
+ }, () -> "OutputStream(UTF-8) path threw for input: " +
describe(text));
+ }
+
+ /** Random string spanning the full BMP: control chars, surrogates,
non-characters. */
+ private static String randomText(Random rng) {
+ char[] buf = new char[rng.nextInt(32)];
+ for (int i = 0; i < buf.length; i++) {
+ buf[i] = (char) rng.nextInt(0x10000);
+ }
+ return new String(buf);
+ }
+
+ /** Hex-escapes a string so a fuzz failure reports the exact offending
input. */
+ private static String describe(String s) {
+ StringBuilder sb = new StringBuilder(s.length() * 6);
+ for (int i = 0; i < s.length(); i++) {
+ sb.append(String.format(Locale.ROOT,"\\u%04x", (int) s.charAt(i)));
+ }
+ return sb.toString();
+ }
}
diff --git a/tika-parent/pom.xml b/tika-parent/pom.xml
index 719b343f44..9a2bb0b5da 100644
--- a/tika-parent/pom.xml
+++ b/tika-parent/pom.xml
@@ -329,6 +329,7 @@
<bouncycastle.version>1.84</bouncycastle.version>
<!-- NOTE: sync brotli version with commons-compress-->
<brotli.version>0.1.2</brotli.version>
+ <commonmark.version>0.29.0</commonmark.version>
<commons.cli.version>1.11.0</commons.cli.version>
<commons.codec.version>1.22.0</commons.codec.version>
<commons.collections4.version>4.5.0</commons.collections4.version>
@@ -1009,6 +1010,21 @@
<artifactId>bcprov-jdk18on</artifactId>
<version>${bouncycastle.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.commonmark</groupId>
+ <artifactId>commonmark</artifactId>
+ <version>${commonmark.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.commonmark</groupId>
+ <artifactId>commonmark-ext-gfm-strikethrough</artifactId>
+ <version>${commonmark.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.commonmark</groupId>
+ <artifactId>commonmark-ext-gfm-tables</artifactId>
+ <version>${commonmark.version}</version>
+ </dependency>
<dependency>
<groupId>org.freemarker</groupId>
<artifactId>freemarker</artifactId>
diff --git a/tika-parsers/tika-parsers-ml/tika-vlm/pom.xml
b/tika-parsers/tika-parsers-ml/tika-vlm/pom.xml
index eeddec6075..409e7ebbc0 100644
--- a/tika-parsers/tika-parsers-ml/tika-vlm/pom.xml
+++ b/tika-parsers/tika-parsers-ml/tika-vlm/pom.xml
@@ -29,10 +29,6 @@
<artifactId>tika-vlm</artifactId>
<name>Apache Tika VLM module</name>
- <properties>
- <commonmark.version>0.29.0</commonmark.version>
- </properties>
-
<dependencies>
<dependency>
<groupId>${project.groupId}</groupId>
@@ -46,17 +42,14 @@
<dependency>
<groupId>org.commonmark</groupId>
<artifactId>commonmark</artifactId>
- <version>${commonmark.version}</version>
</dependency>
<dependency>
<groupId>org.commonmark</groupId>
<artifactId>commonmark-ext-gfm-tables</artifactId>
- <version>${commonmark.version}</version>
</dependency>
<dependency>
<groupId>org.commonmark</groupId>
<artifactId>commonmark-ext-gfm-strikethrough</artifactId>
- <version>${commonmark.version}</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
