This is an automated email from the ASF dual-hosted git repository.
davsclaus pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/camel-website.git
The following commit(s) were added to refs/heads/main by this push:
new 3fe58f7c fix: repair malformed HTML before converting pages to
Markdown (#1750)
3fe58f7c is described below
commit 3fe58f7c1f00fc7fa8c6b69fee91098ec4f90f5a
Author: Adriano Machado <[email protected]>
AuthorDate: Mon Sep 14 02:57:01 2026 -0400
fix: repair malformed HTML before converting pages to Markdown (#1750)
Asciidoctor emits mis-nested inline tags for some sources: a `*` inside
backticks renders as <code>core/camel-<strong></code>...</strong>.
node-html-parser drops the unmatched close tag and, at end of input,
unwraps every unclosed ancestor including article.doc, so
generate-markdown skipped the page silently and its .md mirror 404ed
(35 pages in production).
Pages that node-html-parser's valid() rejects are now repaired with
jsdom's HTML5 parser before extraction, the way browsers do. The
parseNoneClosedTags option is not enough: it keeps the unclosed <code>
open and Turndown flattens the rest of the article into one inline code
span. Well-formed pages take the unchanged path.
Repaired pages and pages without main content are now logged instead of
skipped silently, so the build names the sources to fix upstream.
Fixes #1746
_Claude Code on behalf of Adriano Machado (@ammachado)_
_This was generated by an AI agent and may contain inaccuracies.
Please verify before relying on it._
---
gulp/tasks/generate-markdown.js | 108 +++++++++++++++++++++++++---------------
test/generate-markdown-test.js | 90 +++++++++++++++++++++++++++++++++
2 files changed, 158 insertions(+), 40 deletions(-)
diff --git a/gulp/tasks/generate-markdown.js b/gulp/tasks/generate-markdown.js
index c4c93534..8c210a03 100644
--- a/gulp/tasks/generate-markdown.js
+++ b/gulp/tasks/generate-markdown.js
@@ -1,5 +1,6 @@
const fs = require('fs');
-const { parse } = require('node-html-parser');
+const { JSDOM } = require('jsdom');
+const { parse, valid } = require('node-html-parser');
const { createTurndownService } = require('../helpers/turndown-config');
const { generateToonSitemaps } = require('../helpers/toon-format');
const { generateLlmsTxt } = require('../helpers/llms-txt');
@@ -42,50 +43,17 @@ async function generateMarkdown() {
for (const htmlFile of batch) {
try {
const htmlContent = fs.readFileSync(htmlFile, 'utf8');
- const root = parse(htmlContent);
+ const { markdown, repaired } = convertPage(htmlContent,
turndownService);
- // Extract only the main article content
- // Try different selectors based on Antora and Hugo structure
- let mainContent = root.querySelector('article.doc') ||
- root.querySelector('main') ||
- root.querySelector('.article') ||
- root.querySelector('article');
+ if (repaired) {
+ console.warn(`Repaired malformed HTML in ${htmlFile}, fix the
mis-nested markup in its source`);
+ }
- if (!mainContent) {
- // Silently skip files without main content
+ if (markdown === null) {
+ console.warn(`Skipping ${htmlFile}: no main content found`);
continue;
}
- // Remove navigation elements, headers, and footers from the content
- const elementsToRemove = mainContent.querySelectorAll('nav, header,
footer, .nav, .navbar, .toolbar');
- elementsToRemove.forEach(el => el.remove());
-
- // Remove anchor links (they are just UI navigation aids)
- const anchors = mainContent.querySelectorAll('a.anchor');
- anchors.forEach(el => el.remove());
-
- // Clean up table cells by unwrapping div.content and div.paragraph
wrappers
- const tableCells = mainContent.querySelectorAll('td.tableblock,
th.tableblock');
- tableCells.forEach(cell => {
- let html = cell.innerHTML;
- // Unwrap <div class="content"><div
class="paragraph"><p>...</p></div></div>
- html = html.replace(/<div class="content"><div
class="paragraph">\s*<p>(.*?)<\/p>\s*<\/div><\/div>/gs, '$1');
- // Unwrap <div class="content"><div id="..."
class="paragraph"><p>...</p></div></div>
- html = html.replace(/<div
class="content"><div[^>]*class="paragraph"[^>]*>\s*<p>(.*?)<\/p>\s*<\/div><\/div>/gs,
'$1');
- // Also handle simple <p class="tableblock">...</p> wrappers
- html = html.replace(/<p class="tableblock">(.*?)<\/p>/gs, '$1');
- cell.set_content(html);
- });
-
- // Convert to Markdown
- let markdown = turndownService.turndown(mainContent.innerHTML);
-
- // Update links to point to .md files instead of .html
- // Replace https://camel.apache.org/**/*.html with
https://camel.apache.org/**/*.md
- markdown =
markdown.replace(/(https:\/\/camel\.apache\.org\/[^)\s]*?)\.html/g, '$1.md');
- // Replace relative links *.html with *.md
- markdown = markdown.replace(/\[([^\]]+)\]\(([^)]+?)\.html\)/g,
'[$1]($2.md)');
-
// Write .md file (replace .html extension with .md)
const mdFile = htmlFile.replace(/\.html$/, '.md');
fs.writeFileSync(mdFile, markdown, 'utf8');
@@ -119,4 +87,64 @@ async function generateMarkdown() {
await generateAllIndexes();
}
+/**
+ * Converts the main content of one rendered HTML page to Markdown.
+ *
+ * @param {string} htmlContent the full HTML page
+ * @param {TurndownService} turndownService configured Turndown instance
+ * @returns {{markdown: string|null, repaired: boolean}} the Markdown (null
when the page has no
+ * main content), and whether the HTML was malformed and had to be repaired
before parsing
+ */
+function convertPage(htmlContent, turndownService) {
+ // node-html-parser cannot repair mis-nested inline tags (Asciidoctor emits
them for a `*` inside
+ // backticks): it unwraps every unclosed ancestor, article.doc included. Let
jsdom's HTML5 parser
+ // repair such pages the way browsers do; it is much slower, so only
malformed pages go through it.
+ const repaired = !valid(htmlContent);
+ const root = parse(repaired ? new JSDOM(htmlContent).serialize() :
htmlContent);
+
+ // Extract only the main article content
+ // Try different selectors based on Antora and Hugo structure
+ let mainContent = root.querySelector('article.doc') ||
+ root.querySelector('main') ||
+ root.querySelector('.article') ||
+ root.querySelector('article');
+
+ if (!mainContent) {
+ return { markdown: null, repaired };
+ }
+
+ // Remove navigation elements, headers, and footers from the content
+ const elementsToRemove = mainContent.querySelectorAll('nav, header, footer,
.nav, .navbar, .toolbar');
+ elementsToRemove.forEach(el => el.remove());
+
+ // Remove anchor links (they are just UI navigation aids)
+ const anchors = mainContent.querySelectorAll('a.anchor');
+ anchors.forEach(el => el.remove());
+
+ // Clean up table cells by unwrapping div.content and div.paragraph wrappers
+ const tableCells = mainContent.querySelectorAll('td.tableblock,
th.tableblock');
+ tableCells.forEach(cell => {
+ let html = cell.innerHTML;
+ // Unwrap <div class="content"><div
class="paragraph"><p>...</p></div></div>
+ html = html.replace(/<div class="content"><div
class="paragraph">\s*<p>(.*?)<\/p>\s*<\/div><\/div>/gs, '$1');
+ // Unwrap <div class="content"><div id="..."
class="paragraph"><p>...</p></div></div>
+ html = html.replace(/<div
class="content"><div[^>]*class="paragraph"[^>]*>\s*<p>(.*?)<\/p>\s*<\/div><\/div>/gs,
'$1');
+ // Also handle simple <p class="tableblock">...</p> wrappers
+ html = html.replace(/<p class="tableblock">(.*?)<\/p>/gs, '$1');
+ cell.set_content(html);
+ });
+
+ // Convert to Markdown
+ let markdown = turndownService.turndown(mainContent.innerHTML);
+
+ // Update links to point to .md files instead of .html
+ // Replace https://camel.apache.org/**/*.html with
https://camel.apache.org/**/*.md
+ markdown =
markdown.replace(/(https:\/\/camel\.apache\.org\/[^)\s]*?)\.html/g, '$1.md');
+ // Replace relative links *.html with *.md
+ markdown = markdown.replace(/\[([^\]]+)\]\(([^)]+?)\.html\)/g,
'[$1]($2.md)');
+
+ return { markdown, repaired };
+}
+
module.exports = generateMarkdown;
+module.exports.convertPage = convertPage;
diff --git a/test/generate-markdown-test.js b/test/generate-markdown-test.js
new file mode 100644
index 00000000..e1941714
--- /dev/null
+++ b/test/generate-markdown-test.js
@@ -0,0 +1,90 @@
+'use strict'
+
+const assert = require('node:assert/strict')
+const test = require('node:test')
+
+const { convertPage } = require('../gulp/tasks/generate-markdown')
+const { createTurndownService } = require('../gulp/helpers/turndown-config')
+
+function page (body) {
+ return '<!DOCTYPE html><html><head><title>t</title></head><body><nav
class="nav">menu</nav>' +
+ `<main class="article"><article
class="doc">\n${body}\n</article></main></body></html>`
+}
+
+const LATER_SECTION = `<div class="sect2">
+<h3 id="_later"><a class="anchor" href="#_later"></a>Later section</h3>
+<div class="paragraph">
+<p>Still prose.</p>
+</div>
+</div>`
+
+test('a well-formed page converts its article, dropping anchors and rewriting
.html links', () => {
+ const html = page(`<h2 id="_intro"><a class="anchor"
href="#_intro"></a>Intro</h2>
+<div class="paragraph">
+<p>Use <code>camel run</code>, see <a href="other.html">Other</a>.</p>
+</div>`)
+
+ assert.deepEqual(convertPage(html, createTurndownService()), {
+ markdown: '## Intro\n\nUse `camel run`, see [Other](other.md).',
+ repaired: false,
+ })
+})
+
+test('a page without main content yields no Markdown', () => {
+ const html = '<!DOCTYPE
html><html><head><title>t</title></head><body><p>verification</p></body></html>'
+
+ assert.deepEqual(convertPage(html, createTurndownService()), { markdown:
null, repaired: false })
+})
+
+// NOTE the fixtures below are what Asciidoctor emits for real Camel docs
(security-model.adoc,
+// camel-jbang-mcp.adoc). Browsers repair the mis-nesting, but
node-html-parser drops the unmatched
+// close tag and, at end of input, unwraps every unclosed ancestor including
article.doc, so the
+// page used to be skipped and its .md mirror 404ed. Asserting that the page
merely converts is not
+// enough: keeping the unclosed element open instead (parseNoneClosedTags)
retains article.doc but
+// nests everything after the glitch inside <code>, which Turndown flattens
into one inline code
+// span. The later heading and paragraph must survive as their own Markdown
lines.
+test('a wildcard read as bold inside backticks keeps the page and the sections
after it', () => {
+ const html = page(`<div class="paragraph">
+<p>A candidate located in a <code>core/camel-<strong></code> module is judged
against these
+invariants first. If the engine upheld the invariant *and</strong> nothing
else.</p>
+</div>
+${LATER_SECTION}`)
+
+ const { markdown } = convertPage(html, createTurndownService())
+
+ assert.notEqual(markdown, null)
+ assert.match(markdown, /module is judged against these/)
+ assert.match(markdown, /^### Later section$/m)
+ assert.match(markdown, /^Still prose\.$/m)
+})
+
+test('an unterminated code span inside emphasis keeps the list items after
it', () => {
+ const html = page(`<div class="ulist">
+<ul>
+<li>
+<p><em>"Validate this endpoint:
<code>kafka:myTopic?brkers=localhost:9092\`"</em> - detects the typo and
suggests \`brokers</code></p>
+</li>
+<li>
+<p><em>"Validate this YAML route"</em> - checks against the YAML DSL JSON
schema</p>
+</li>
+</ul>
+</div>
+${LATER_SECTION}`)
+
+ const { markdown } = convertPage(html, createTurndownService())
+
+ assert.notEqual(markdown, null)
+ assert.match(markdown, /^-\s+_"Validate this YAML route"_ - checks against
the YAML DSL JSON schema$/m)
+ assert.match(markdown, /^### Later section$/m)
+})
+
+// NOTE the repair hides the defect from readers of the .md mirror, but the
rendered HTML is still
+// invalid and the fix belongs in the upstream .adoc source. Flagging the page
is what lets the build
+// log name it.
+test('a page that needed repair is flagged so its source can be fixed
upstream', () => {
+ const html = page(`<div class="paragraph">
+<p>A <code>core/camel-<strong></code> module</strong> x</p>
+</div>`)
+
+ assert.equal(convertPage(html, createTurndownService()).repaired, true)
+})
