LiteSun commented on code in PR #2078:
URL: https://github.com/apache/apisix-website/pull/2078#discussion_r3654016849
##########
next/scripts/generate-md-twins.mjs:
##########
@@ -0,0 +1,160 @@
+/**
+ * Post-build agent-readable surfaces.
+ *
+ * For every content page the build produced, emit a Markdown twin next to the
+ * HTML (`<page>/index.md`) and index them all in `/llms.txt`. Agents that read
+ * docs — and the crawlers behind them — get clean prose instead of parsing a
+ * page of markup.
+ *
+ * The twin is the *synced source* markdown, with frontmatter replaced by a
+ * title heading and a link back to the canonical HTML. Source markdown lives
+ * in content/ (written by sync-content.mjs), so this runs after `astro build`
+ * and needs no MDX evaluation.
+ */
+import fs from 'node:fs';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+const root = path.dirname(path.dirname(fileURLToPath(import.meta.url)));
+const args = process.argv.slice(2);
+const distFlag = args.indexOf('--dist');
+const dist = distFlag !== -1 ? path.resolve(args[distFlag + 1]) :
path.join(root, 'dist');
+const content = path.join(root, 'content');
+const SITE = 'https://apisix.apache.org';
+
+/**
+ * content/ subdir -> every URL prefix its pages are published under. Several
+ * collections have one source that renders at both locales (the zh site falls
+ * back to the English text where no translation exists), so a source file can
+ * legitimately map to two URLs.
+ */
+const COLLECTIONS = [
+ ['blog-en', ['/blog']],
+ ['blog-zh', ['/zh/blog']],
+ ['learning-center', ['/learning-center', '/zh/learning-center']],
+ ['articles', ['/articles', '/zh/articles']],
+ ['docs-general', ['/docs/general', '/zh/docs/general']],
+ ['docs-apisix-en', ['/docs/apisix']],
Review Comment:
This mapping does not cover the English fallback used by the Chinese APISIX
docs. When a page has no Chinese translation, the build still publishes an
English-backed page under `/zh/docs/apisix/...`, but `docs-apisix-en` is only
checked against `/docs/apisix`. In the release build this leaves 21 published
Chinese-locale HTML pages without the promised `index.md` twin. Please include
the Chinese prefix as a fallback for this collection, or add a parity assertion
that every migrated HTML page has a twin.
##########
.github/workflows/deploy.yml:
##########
@@ -229,6 +315,72 @@ jobs:
mkdir -p "website/build/$(dirname "$path")"
cp -R "next/dist/$path" "website/build/$path"
done
+ # Latest-version docs come from Astro; every versioned directory
+ # (3.16/, next/, v1.6/, …) stays exactly as Docusaurus built it.
+ # Replacement is per-page, not per-subtree: under docs/<project>/ the
+ # version dirs and the latest-version pages sit side by side, so a
+ # subtree swap would delete the archive. Only paths the Astro build
+ # actually produced are touched.
+ for locale_prefix in "" "zh/"; do
+ src="next/dist/${locale_prefix}docs"
+ [ -d "$src" ] || continue
+ (cd "$src" && find . -name index.html) | sed 's|^\./||' | while
read -r rel; do
+ case "$rel" in
+ # Never touch an archived version directory.
+ */[0-9].[0-9]*/*|*/next/*|*/v[0-9]*/*) continue ;;
+ esac
+ dest="website/build/${locale_prefix}docs/$rel"
+ mkdir -p "$(dirname "$dest")"
+ cp "$src/$rel" "$dest"
+ # Ship the Markdown twin next to the page it mirrors.
+ twin="${rel%index.html}index.md"
+ [ -f "$src/$twin" ] && cp "$src/$twin"
"website/build/${locale_prefix}docs/$twin"
+ done
+ done
+ # The version archives must survive untouched.
+ test -f website/build/docs/apisix/3.16/plugins/cors/index.html
+ test -f website/build/docs/apisix/next/plugins/cors/index.html
+ grep -q 'docusaurus'
website/build/docs/apisix/3.16/plugins/cors/index.html
+ # …and the latest-version pages must now be the Astro build.
+ grep -q '/_astro/' website/build/docs/apisix/plugins/cors/index.html
+ grep -q '/_astro/'
website/build/zh/docs/apisix/plugins/cors/index.html
+ # The archives are still a Docusaurus SPA, and their version dropdown
+ # and "older version" banner link to the version-less URLs we just
+ # replaced. Without a real navigation, React would client-route to
+ # its own stale render of those URLs instead of fetching the Astro
+ # page. target="_self" forces a document load (same trick the repo
+ # already uses for pathname:// links in the locale dropdown).
+ node -e '
+ const fs = require("fs"), path = require("path");
+ const roots = ["website/build/docs", "website/build/zh/docs"];
+ // /docs/<project>/<page…>/ with no version segment right after the
+ // project — i.e. exactly the URLs the Astro build now owns.
+ const re =
/(<a\b[^>]*?\bhref="\/(?:zh\/)?docs\/[a-z0-9-]+\/(?!(?:[0-9]+\.[0-9]+|next|v[0-9])\/)[^"]*")/g;
+ let patched = 0;
+ const walk = (d) => fs.existsSync(d) && fs.readdirSync(d, {
withFileTypes: true }).forEach((e) => {
+ const p = path.join(d, e.name);
+ if (e.isDirectory()) return walk(p);
+ if (e.name !== "index.html") return;
+ // Only the archived pages need this; the Astro pages are not a
SPA.
+ const html = fs.readFileSync(p, "utf8");
+ if (!html.includes("docusaurus")) return;
+ const out = html.replace(re, (m) => (m.includes("target=") ? m :
`${m} target="_self"`));
Review Comment:
Adding `target="_self"` does not force a document navigation in the hydrated
Docusaurus app. Docusaurus renders internal URLs through React Router, and
React Router still intercepts clicks when the target is absent or `_self`,
calls `preventDefault()`, and performs client-side history navigation. Patching
the generated HTML also does not change the link props React hydrates with. The
archive dropdown/banner can therefore still render Docusaurus's stale
versionless page instead of fetching the Astro page. Please use a URL/link form
that Docusaurus treats as external, or add an explicit hard-navigation handler,
and verify the behavior after hydration rather than only asserting the
serialized HTML.
##########
next/scripts/generate-md-twins.mjs:
##########
@@ -0,0 +1,160 @@
+/**
+ * Post-build agent-readable surfaces.
+ *
+ * For every content page the build produced, emit a Markdown twin next to the
+ * HTML (`<page>/index.md`) and index them all in `/llms.txt`. Agents that read
+ * docs — and the crawlers behind them — get clean prose instead of parsing a
+ * page of markup.
+ *
+ * The twin is the *synced source* markdown, with frontmatter replaced by a
+ * title heading and a link back to the canonical HTML. Source markdown lives
+ * in content/ (written by sync-content.mjs), so this runs after `astro build`
+ * and needs no MDX evaluation.
+ */
+import fs from 'node:fs';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+const root = path.dirname(path.dirname(fileURLToPath(import.meta.url)));
+const args = process.argv.slice(2);
+const distFlag = args.indexOf('--dist');
+const dist = distFlag !== -1 ? path.resolve(args[distFlag + 1]) :
path.join(root, 'dist');
+const content = path.join(root, 'content');
+const SITE = 'https://apisix.apache.org';
+
+/**
+ * content/ subdir -> every URL prefix its pages are published under. Several
+ * collections have one source that renders at both locales (the zh site falls
+ * back to the English text where no translation exists), so a source file can
+ * legitimately map to two URLs.
+ */
+const COLLECTIONS = [
+ ['blog-en', ['/blog']],
+ ['blog-zh', ['/zh/blog']],
+ ['learning-center', ['/learning-center', '/zh/learning-center']],
+ ['articles', ['/articles', '/zh/articles']],
+ ['docs-general', ['/docs/general', '/zh/docs/general']],
+ ['docs-apisix-en', ['/docs/apisix']],
+ ['docs-apisix-zh', ['/zh/docs/apisix']],
+];
+for (const p of ['ingress-controller', 'helm-chart', 'docker',
'java-plugin-runner', 'go-plugin-runner', 'python-plugin-runner']) {
+ COLLECTIONS.push([`docs-${p}-en`, [`/docs/${p}`, `/zh/docs/${p}`]]);
+ COLLECTIONS.push([`docs-${p}-zh`, [`/zh/docs/${p}`]]);
+}
+
+const walk = (d) => (fs.existsSync(d)
+ ? fs.readdirSync(d, { withFileTypes: true }).flatMap((e) => {
+ const p = path.join(d, e.name);
+ return e.isDirectory() ? walk(p) : (e.name.endsWith('.md') ? [p] : []);
+ })
+ : []);
+
+/** Strip frontmatter, returning it parsed shallowly plus the body. */
+function splitFrontmatter(src) {
+ const m = src.match(/^---\r?\n([\s\S]*?)\r?\n---\r?\n?/);
+ if (!m) return { fm: {}, body: src };
+ const fm = {};
+ for (const line of m[1].split('\n')) {
+ const kv = line.match(/^(\w[\w-]*):\s*(.*)$/);
+ if (kv) fm[kv[1]] = kv[2].replace(/^["']|["']$/g, '').trim();
+ }
+ return { fm, body: src.slice(m[0].length) };
+}
+
+/**
+ * Map a source file to the URL its page was published at, by finding the built
+ * HTML. Slug/id frontmatter overrides and date-based blog paths mean the file
+ * path alone is not authoritative, so candidates are checked against dist/.
+ */
+function resolveUrl(file, collectionDir, urlPrefix) {
+ const rel = path.relative(path.join(content, collectionDir), file)
+ .replace(/\.md$/, '')
+ .split(path.sep).join('/');
+ const { fm } = splitFrontmatter(fs.readFileSync(file, 'utf8'));
+ const candidates = [];
+
+ if (collectionDir.startsWith('blog-')) {
+ // Blog: /blog/YYYY/MM/DD/<name>/, or a slug override replacing the path.
+ const m = rel.match(/^(\d{4})\/(\d{2})\/(\d{2})\/(.+)$/);
+ const slug = (fm.slug || '').replace(/^\/+/, '');
+ if (slug && slug.includes('/')) candidates.push(`${urlPrefix}/${slug}/`);
+ if (m) candidates.push(`${urlPrefix}/${m[1]}/${m[2]}/${m[3]}/${slug ||
m[4]}/`);
+ } else {
+ const id = (fm.slug || fm.id || '').replace(/^\/+/, '');
+ const dir = rel.includes('/') ? `${rel.slice(0, rel.lastIndexOf('/'))}/` :
'';
+ if (id) candidates.push(`${urlPrefix}/${id.includes('/') ? id : dir +
id}/`);
+ candidates.push(`${urlPrefix}/${rel}/`);
+ }
+
+ for (const url of candidates) {
+ if (fs.existsSync(path.join(dist, url, 'index.html'))) return url;
+ }
+ return null;
+}
+
+const written = [];
+let skipped = 0;
+
+for (const [dir, urlPrefixes] of COLLECTIONS) {
+ for (const file of walk(path.join(content, dir))) {
+ const src = fs.readFileSync(file, 'utf8');
+ const { fm, body } = splitFrontmatter(src);
+ const title = fm.title || path.basename(file, '.md');
+ let matched = false;
+
+ for (const urlPrefix of urlPrefixes) {
+ const url = resolveUrl(file, dir, urlPrefix);
+ if (!url) continue;
+ matched = true;
+ // Upstream files open with the ASF licence header as an HTML comment;
+ // it is legal boilerplate, not content, and only wastes an agent's
+ // context window. The licence still ships with the source repo.
+ const prose = body.replace(/^\s*<!--[\s\S]*?-->\s*/, '').trim();
+ const doc = [
+ `# ${title}`,
+ '',
+ fm.description ? `> ${fm.description}` : null,
+ fm.description ? '' : null,
+ `Source: ${SITE}${url}`,
+ '',
+ prose,
+ '',
+ ].filter((l) => l !== null).join('\n');
+ fs.writeFileSync(path.join(dist, url, 'index.md'), doc);
+ written.push({ url, title, description: fm.description || '' });
+ }
+ if (!matched) skipped += 1;
+ }
+}
+
+// /llms.txt — the index agents fetch first.
+written.sort((a, b) => a.url.localeCompare(b.url));
+const section = (label, items) => (items.length
+ ? [`## ${label}`, '', ...items.map((p) => `-
[${p.title}](${SITE}${p.url}index.md)${p.description ? ` — ${p.description}` :
''}`), '']
+ : []);
+
+const isZh = (p) => p.url.startsWith('/zh/');
+const en = written.filter((p) => !isZh(p));
+const zh = written.filter(isZh);
+const group = (items, frag) => items.filter((p) => p.url.includes(frag));
+
+const llms = [
+ '# Apache APISIX',
+ '',
+ '> Apache APISIX is a dynamic, real-time, high-performance API gateway and
AI gateway.',
+ '',
+ 'Every page below is available as Markdown — append `index.md` to any page
URL.',
+ '',
+ ...section('Documentation', group(en, '/docs/')),
+ ...section('Learning center', group(en, '/learning-center/')),
+ ...section('Blog', group(en, '/blog/')),
+ ...section('Articles', group(en, '/articles/')),
+ ...section('中文文档', group(zh, '/docs/')),
+ ...section('中文学习中心', group(zh, '/learning-center/')),
+ ...section('中文博客', group(zh, '/blog/')),
Review Comment:
The `articles` collection generates both English and Chinese-locale twins,
but the index has no Chinese articles section. The generated
`/zh/articles/.../index.md` files (13 in the current build) are therefore
omitted from `/llms.txt`. Please add the corresponding section so the index
covers the twins this script publishes.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
