dabla commented on code in PR #74536: URL: https://github.com/apache/airflow/pull/74536#discussion_r4306469982
########## devel-common/src/sphinx_exts/airflow_llms_txt.py: ########## @@ -0,0 +1,533 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Publish Markdown copies of each page and an ``llms.txt`` index for LLMs and coding agents. + +`sphinx-llm <https://github.com/NVIDIA/sphinx-llm>`__ does the heavy lifting: it runs a second +Sphinx build with a Markdown builder, writes ``page.html.md`` next to every ``page.html`` and +adds ``<link rel="alternate" type="text/markdown">`` to each HTML page. This extension adds what +Airflow docs need on top of it: + +* Markdown handlers for nodes the stock translator drops or misrenders: ``mermaid`` diagrams, + code-block captions, keyword-only ``*`` markers in API signatures and toctree captions. +* License-header comments are removed, so they do not open every page. +* ``exampleinclude`` headers link to the full example file on GitHub at the release tag, since + the excerpts leave out imports. +* ``llms.txt`` is rewritten: sections follow the root toctree captions, every link is absolute, + and descriptions come from each page's first paragraph rather than its first line of text. +* ``llms-full.txt`` holds the same pages in the same order, each headed by its URL, with links + made absolute so they still work once the pages share one file. +""" + +from __future__ import annotations + +import posixpath +import re +from collections.abc import Iterator +from pathlib import Path +from typing import TYPE_CHECKING, Any, cast +from urllib.parse import urljoin, urlsplit, urlunsplit + +from docutils import nodes +from sphinx import addnodes +from sphinx.errors import ExtensionError +from sphinx.util import logging +from sphinx.util.matching import Matcher +from sphinx_llm.markdown_builder import SphinxLlmMarkdownBuilder, SphinxLlmMarkdownTranslator +from sphinx_markdown_builder.contexts import IndentContext, SubContext, SubContextParams, TableContext + +from sphinx_exts.redirects import iter_redirects + +if TYPE_CHECKING: + from sphinx.application import Sphinx + from sphinx.config import Config + from sphinx.domains.std import StandardDomain + from sphinx.environment import BuildEnvironment + +logger = logging.getLogger(__name__) + +MARKDOWN_BUILDER = SphinxLlmMarkdownBuilder.name +# Pages whose first paragraph is shorter than this get no description rather than a fragment. +MIN_DESCRIPTION_LENGTH = 20 +MAX_DESCRIPTION_LENGTH = 240 +_SENTENCE_END = re.compile(r"(?<=[.!?])\s+(?=[A-Z`])") +# Paragraphs under these are not the page's own opening prose. +_NOT_PROSE = ( + nodes.Admonition, + nodes.comment, + nodes.compound, + nodes.list_item, + nodes.system_message, + nodes.table, + nodes.topic, + addnodes.desc, +) +# llms.txt is built from the HTML build's doctrees, where Sphinx has already curled the quotes. +# The files are served without a charset, so keep them ASCII (see SMARTQUOTES_EXCLUDES). +_TYPOGRAPHY = str.maketrans( + { + "\u2018": "'", + "\u2019": "'", + "\u201c": '"', + "\u201d": '"', + "\u2013": "-", + "\u2014": "-", + "\u2026": "...", + "\u00a0": " ", + } +) +_INLINE_CODE = re.compile(r"(`[^`]*`)") +_MARKDOWN_LINK = re.compile(r"(\]\()([^)\s]+)(\))") + + +class CompactTableContext(TableContext): + """ + A pipe table without the column padding. + + ``tabulate`` pads every cell to the widest one in its column, which is pure whitespace to a + model reading the file: about a tenth of ``llms-full.txt`` before this. + """ + + def make(self) -> str: + ctx = SubContext() + prefix = self.internal_context.make() + if prefix: + ctx.add(prefix) + rows = [self.make_row(row) for row in [*self.headers, *self.body]] + if rows: + width = max(len(row) for row in rows) + lines = [ + "| " + + " | ".join(cell.replace("|", r"\|") for cell in [*row, *[""] * (width - len(row))]) + + " |" + for row in rows + ] + lines.insert(1, "|" + "---|" * width) + ctx.add("\n".join(lines), prefix_eol=2) + return ctx.make() + + +class AirflowMarkdownTranslator(SphinxLlmMarkdownTranslator): + """Render the Sphinx nodes Airflow docs use that sphinx-markdown-builder skips or misrenders.""" + + def _push_box(self, title: str) -> None: + # Admonitions become a quoted block that starts with their label, rather than an + # ``#### NOTE`` heading that breaks the heading order and never says where the note ends. + self._push_context(IndentContext("> ", empty=True, params=SubContextParams(2, 2))) + self.add(f"**{title.capitalize()}:**", suffix_eol=2) + + def _fetch_ref_uri(self, node: nodes.reference) -> str: + # A link to a section on the same page (```Confidence`_``) has a refid but no refuri and + # no ``internal`` flag, so the base translators leave its target empty. + if node.get("refid") and not node.get("refuri"): + return self.builder.link_token(self.builder.current_doc_name, node["refid"]) + return super()._fetch_ref_uri(node) + + def visit_meta(self, node: nodes.Element) -> None: + raise nodes.SkipNode + + def visit_toctree(self, node: addnodes.toctree) -> None: + # Hidden toctrees can reach the writer unresolved; like HTML, show nothing for them. + # Visible ones are resolved into link lists before this, so any left are unexpected. + if not node.get("hidden"): + super().unknown_visit(node) + raise nodes.SkipNode + + def visit_table(self, node: nodes.Element) -> None: + self._push_context(CompactTableContext(params=SubContextParams(2, 1))) + + def depart_table(self, node: nodes.Element) -> None: + self._pop_context(node) + + def visit_mermaid(self, node: nodes.Element) -> None: + self.add(f"```mermaid\n{node['code'].strip()}\n```", prefix_eol=2, suffix_eol=2) + raise nodes.SkipNode + + def visit_caption(self, node: nodes.Element) -> None: + self.add(f"**{node.astext()}**", prefix_eol=2, suffix_eol=1) + raise nodes.SkipNode + + # ``abbreviation`` wraps the bare ``*`` that marks keyword-only parameters in signatures. + def visit_abbreviation(self, node: nodes.Element) -> None: + pass + + def depart_abbreviation(self, node: nodes.Element) -> None: + pass + + def visit_title(self, node: nodes.Element) -> None: + # Render navigation captions in bold rather than as section headings. + parent = node.parent + if (isinstance(parent, addnodes.compact_paragraph) and parent.get("toctree")) or ( + isinstance(parent, nodes.compound) and "toctree-wrapper" in parent.get("classes", []) + ): + self.add(f"**{node.astext()}**", prefix_eol=2, suffix_eol=1) + raise nodes.SkipNode + + super().visit_title(node) + + def depart_title(self, node: nodes.Element) -> None: + self._pop_context(node) + + +def _escape_link_text(text: str) -> str: + return text.replace("[", r"\[").replace("]", r"\]") + + +def _inline_markdown(node: nodes.Node, env: BuildEnvironment) -> str: + """Flatten a title or paragraph to one line of Markdown, keeping inline code as code.""" + parts: list[str] = [] + for child in node.children: + if isinstance(child, addnodes.pending_xref) and not child.get("refexplicit"): + parts.append(_xref_title(child, env) or _inline_markdown(child, env)) + elif isinstance(child, nodes.literal): + parts.append(f"`{child.astext()}`") + elif isinstance(child, nodes.Text): + parts.append(child.astext()) + elif isinstance(child, nodes.Element): + parts.append(_inline_markdown(child, env)) + return re.sub(r"\s+", " ", "".join(parts)).strip().translate(_TYPOGRAPHY) + + +def _xref_title(xref: addnodes.pending_xref, env: BuildEnvironment) -> str | None: + """ + Return the title a ``:doc:`` or ``:ref:`` without explicit text resolves to. + + Doctrees read back from the environment are unresolved, so such a reference still holds its + raw target (``operators/llm``) rather than the page or section title the HTML shows. + """ + target = xref.get("reftarget", "") + if xref.get("reftype") == "doc": + refdoc = xref.get("refdoc", "") + docname = ( + target.lstrip("/") + if target.startswith("/") + else posixpath.join(posixpath.dirname(refdoc), target) + ) + title = env.titles.get(posixpath.normpath(docname)) + return _inline_markdown(title, env) if title else None + if xref.get("reftype") == "ref": + domain = cast("StandardDomain", env.get_domain("std")) + label = domain.labels.get(target.lower()) + return label[2] if label else None + return None + + +def _separate_markdown_doctrees(app: Sphinx, config: Config) -> None: + """Give the Markdown build its own doctree cache.""" + # sphinx-llm's sequential build shares the HTML cache, which can reuse + # incompatible doctrees or overwrite them. Set a separate cache before + # Sphinx creates the environment. + + if app.tags.has("sphinx_llm_markdown"): + app.doctreedir = Path(app.outdir) / ".doctrees" + + +def _is_markdown_build(app: Sphinx) -> bool: + return app.builder is not None and app.builder.name == MARKDOWN_BUILDER + + +def _link_example_sources(app: Sphinx, doctree: nodes.document) -> None: + """ + Replace each ``exampleinclude`` header with a link to the whole example file. + + Runs before ``exampleinclude``'s own ``doctree-read`` handler, which would otherwise turn the + header into a bare module path plus a "View Source" button that has no Markdown equivalent. + """ + if not _is_markdown_build(app): + return + # Imported by bare name, as conf.py loads it: another import path gives a different class. + from exampleinclude import ExampleHeader + + repo_root = Path(app.config.llms_txt_repo_root).resolve() + source_url = app.config.llms_txt_source_url.rstrip("/") + for header in list(doctree.findall(ExampleHeader)): + path = Path(header["filename"]).resolve() + if not source_url or not path.is_relative_to(repo_root): + header.parent.remove(header) + continue + repo_path = path.relative_to(repo_root).as_posix() + paragraph = nodes.paragraph() + paragraph += nodes.Text("Excerpt from ") + link = nodes.reference("", "", refuri=f"{source_url}/{repo_path}", internal=False) + link += nodes.literal(repo_path, repo_path) + paragraph += link + paragraph += nodes.Text(":") + header.replace_self(paragraph) + + +def _clean_markdown_doctree(app: Sphinx, doctree: nodes.document, docname: str) -> None: + """Remove RST comments and normalize version, GitHub source and site-root links for Markdown.""" + + if not _is_markdown_build(app): + return + site_root = urljoin(app.config.llms_txt_site_url, "/") + release_url = app.config.llms_txt_release_source_url + source_url = app.config.llms_txt_source_url + for reference in doctree.findall(nodes.reference): + refuri = reference.get("refuri", "") + if "|version|" in refuri: + refuri = reference["refuri"] = refuri.replace("|version|", app.config.version) + if release_url and source_url and release_url != source_url: + for kind in ("blob", "tree"): + prefix = release_url.replace("/blob/", f"/{kind}/") + if refuri.startswith(prefix): + refuri = reference["refuri"] = ( + source_url.replace("/blob/", f"/{kind}/") + refuri[len(prefix) :] + ) + # Links into other packages' docs are site-root relative, which means nothing to a model + # that got the page without its URL. + if refuri.startswith("/") and not refuri.startswith("//"): + reference["refuri"] = urljoin(site_root, refuri) + for comment in list(doctree.findall(nodes.comment)): + comment.parent.remove(comment) + for quote in list(doctree.findall(nodes.block_quote)): + if not quote.children: + quote.parent.remove(quote) + + +def _page_description(env: BuildEnvironment, docname: str) -> str: + """Return the page's meta description or the opening sentences of its first paragraph.""" + doctree = env.get_doctree(docname) + for meta in doctree.findall(nodes.meta): + if meta.get("name") == "description" and meta.get("content"): + return meta["content"] + for paragraph in doctree.findall(nodes.paragraph): + # ``example-header`` is the file path ``exampleinclude`` puts above a code excerpt. + if "example-header" in paragraph["classes"] or any( + isinstance(ancestor, _NOT_PROSE) for ancestor in _ancestors(paragraph) + ): + continue + text = _inline_markdown(paragraph, env) + if len(text) >= MIN_DESCRIPTION_LENGTH: + return _summarize(text) + return "" + + +def _summarize(text: str) -> str: + """Return whole sentences of ``text`` up to the length cap, cutting the first one if it is longer.""" + description = "" + for sentence in _SENTENCE_END.split(text): + if description and len(description) + len(sentence) + 1 > MAX_DESCRIPTION_LENGTH: + break + description = f"{description} {sentence}".strip() + if len(description) > MAX_DESCRIPTION_LENGTH: + description = description[:MAX_DESCRIPTION_LENGTH].rsplit(" ", 1)[0] + " ..." + # A paragraph that introduces a list or code block ends on a colon. + return re.sub(r":$", ".", description) + + +def _ancestors(node: nodes.Node) -> Iterator[nodes.Element]: + parent = node.parent + while parent is not None: + yield parent + parent = parent.parent + + +def _page_title(env: BuildEnvironment, docname: str, explicit_title: str | None) -> str: + if explicit_title: + return explicit_title + title_node = env.titles.get(docname) + return _inline_markdown(title_node, env) if title_node else docname + + +def _toctree_docs(env: BuildEnvironment, docname: str, seen: set[str]) -> list[tuple[str, str | None]]: + """Return ``docname`` and everything below it in toctree order, each once.""" + if docname in seen: + return [] + seen.add(docname) + docs: list[tuple[str, str | None]] = [(docname, None)] + for child in env.toctree_includes.get(docname, []): + docs.extend(_toctree_docs(env, child, seen)) + return docs + + +def _write_markdown_redirects(app: Sphinx) -> None: + """Write Markdown stubs for redirected documentation pages.""" + outdir = Path(app.outdir) + + for from_path, to_path in iter_redirects(app): + # Preserve Markdown belonging to an actual source document. + docname = from_path.removesuffix(".html") + if docname in app.env.found_docs: + continue + + target = urlsplit(to_path) + resolved_path = posixpath.normpath(posixpath.join(posixpath.dirname(from_path), target.path)) + + # Only change links within this package to Markdown. + # Other packages and external sites might only publish HTML. + if ( + not target.scheme + and not target.netloc + and not target.path.startswith("/") + and resolved_path != ".." + and not resolved_path.startswith("../") + and target.path.endswith(".html") + ): + target = target._replace(path=f"{target.path}.md") + + destination = urlunsplit(target) + output = outdir / f"{from_path}.md" + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text( + f"# Page moved\n\nThis page has moved to [the new page](<{destination}>).\n", + encoding="utf-8", + ) + + +def _check_markdown_coverage(outdir: Path) -> None: + """Fail if an eligible HTML page has no Markdown copy.""" + excluded = {"genindex.html", "search.html", "py-modindex.html"} + missing = [] + + for page in sorted(outdir.rglob("*.html")): + relative = page.relative_to(outdir) + if "_modules" in relative.parts or page.name in excluded: + continue + if not page.with_name(page.name + ".md").is_file(): + missing.append(relative.as_posix()) + + if missing: + raise ExtensionError( + "HTML pages missing Markdown copies:\n" + "\n".join(f" {page}" for page in missing) + ) + + +def _write_llms_txt(app: Sphinx, exception: Exception | None) -> None: + """ + Replace sphinx-llm's flat ``llms.txt`` with one grouped by the root toctree captions. + + Runs after sphinx-llm has copied the Markdown pages into the HTML output (priority 101). + """ + if exception is not None or app.builder.name != "html" or not app.config.llms_txt_enabled: + return + env = app.env + outdir = Path(app.outdir) + + _write_markdown_redirects(app) + _check_markdown_coverage(outdir) Review Comment: **[nit]** When the whole Markdown build fails, this error buries the cause. When sphinx-llm's Markdown subprocess exits non-zero, `combine_builds` logs an error and returns without copying anything (sphinx-llm 1.1.0, `txt.py`). The coverage check then raises `HTML pages missing Markdown copies:` followed by every page of the package, about 230 lines for common.ai. The subprocess's own error is somewhere above that list. Failing the build is right; the PoC's warning-and-return let it pass with no Markdown at all. Consider a separate, short error when the root page has no `.html.md`, for example "the Markdown build produced no pages; see the sphinx-llm errors above". Keep the per-page list for the partial case it is meant for. ########## devel-common/src/sphinx_exts/airflow_llms_txt.py: ########## @@ -0,0 +1,533 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Publish Markdown copies of each page and an ``llms.txt`` index for LLMs and coding agents. + +`sphinx-llm <https://github.com/NVIDIA/sphinx-llm>`__ does the heavy lifting: it runs a second +Sphinx build with a Markdown builder, writes ``page.html.md`` next to every ``page.html`` and +adds ``<link rel="alternate" type="text/markdown">`` to each HTML page. This extension adds what +Airflow docs need on top of it: + +* Markdown handlers for nodes the stock translator drops or misrenders: ``mermaid`` diagrams, + code-block captions, keyword-only ``*`` markers in API signatures and toctree captions. +* License-header comments are removed, so they do not open every page. +* ``exampleinclude`` headers link to the full example file on GitHub at the release tag, since + the excerpts leave out imports. +* ``llms.txt`` is rewritten: sections follow the root toctree captions, every link is absolute, + and descriptions come from each page's first paragraph rather than its first line of text. +* ``llms-full.txt`` holds the same pages in the same order, each headed by its URL, with links + made absolute so they still work once the pages share one file. +""" + +from __future__ import annotations + +import posixpath +import re +from collections.abc import Iterator +from pathlib import Path +from typing import TYPE_CHECKING, Any, cast +from urllib.parse import urljoin, urlsplit, urlunsplit + +from docutils import nodes +from sphinx import addnodes +from sphinx.errors import ExtensionError +from sphinx.util import logging +from sphinx.util.matching import Matcher +from sphinx_llm.markdown_builder import SphinxLlmMarkdownBuilder, SphinxLlmMarkdownTranslator +from sphinx_markdown_builder.contexts import IndentContext, SubContext, SubContextParams, TableContext + +from sphinx_exts.redirects import iter_redirects + +if TYPE_CHECKING: + from sphinx.application import Sphinx + from sphinx.config import Config + from sphinx.domains.std import StandardDomain + from sphinx.environment import BuildEnvironment + +logger = logging.getLogger(__name__) + +MARKDOWN_BUILDER = SphinxLlmMarkdownBuilder.name +# Pages whose first paragraph is shorter than this get no description rather than a fragment. +MIN_DESCRIPTION_LENGTH = 20 +MAX_DESCRIPTION_LENGTH = 240 +_SENTENCE_END = re.compile(r"(?<=[.!?])\s+(?=[A-Z`])") +# Paragraphs under these are not the page's own opening prose. +_NOT_PROSE = ( + nodes.Admonition, + nodes.comment, + nodes.compound, + nodes.list_item, + nodes.system_message, + nodes.table, + nodes.topic, + addnodes.desc, +) +# llms.txt is built from the HTML build's doctrees, where Sphinx has already curled the quotes. +# The files are served without a charset, so keep them ASCII (see SMARTQUOTES_EXCLUDES). +_TYPOGRAPHY = str.maketrans( + { + "\u2018": "'", + "\u2019": "'", + "\u201c": '"', + "\u201d": '"', + "\u2013": "-", + "\u2014": "-", + "\u2026": "...", + "\u00a0": " ", + } +) +_INLINE_CODE = re.compile(r"(`[^`]*`)") +_MARKDOWN_LINK = re.compile(r"(\]\()([^)\s]+)(\))") + + +class CompactTableContext(TableContext): + """ + A pipe table without the column padding. + + ``tabulate`` pads every cell to the widest one in its column, which is pure whitespace to a + model reading the file: about a tenth of ``llms-full.txt`` before this. + """ + + def make(self) -> str: + ctx = SubContext() + prefix = self.internal_context.make() + if prefix: + ctx.add(prefix) + rows = [self.make_row(row) for row in [*self.headers, *self.body]] + if rows: + width = max(len(row) for row in rows) + lines = [ + "| " + + " | ".join(cell.replace("|", r"\|") for cell in [*row, *[""] * (width - len(row))]) + + " |" + for row in rows + ] + lines.insert(1, "|" + "---|" * width) + ctx.add("\n".join(lines), prefix_eol=2) + return ctx.make() + + +class AirflowMarkdownTranslator(SphinxLlmMarkdownTranslator): + """Render the Sphinx nodes Airflow docs use that sphinx-markdown-builder skips or misrenders.""" + + def _push_box(self, title: str) -> None: + # Admonitions become a quoted block that starts with their label, rather than an + # ``#### NOTE`` heading that breaks the heading order and never says where the note ends. + self._push_context(IndentContext("> ", empty=True, params=SubContextParams(2, 2))) + self.add(f"**{title.capitalize()}:**", suffix_eol=2) + + def _fetch_ref_uri(self, node: nodes.reference) -> str: + # A link to a section on the same page (```Confidence`_``) has a refid but no refuri and + # no ``internal`` flag, so the base translators leave its target empty. + if node.get("refid") and not node.get("refuri"): + return self.builder.link_token(self.builder.current_doc_name, node["refid"]) + return super()._fetch_ref_uri(node) + + def visit_meta(self, node: nodes.Element) -> None: + raise nodes.SkipNode + + def visit_toctree(self, node: addnodes.toctree) -> None: + # Hidden toctrees can reach the writer unresolved; like HTML, show nothing for them. + # Visible ones are resolved into link lists before this, so any left are unexpected. + if not node.get("hidden"): + super().unknown_visit(node) + raise nodes.SkipNode + + def visit_table(self, node: nodes.Element) -> None: + self._push_context(CompactTableContext(params=SubContextParams(2, 1))) + + def depart_table(self, node: nodes.Element) -> None: + self._pop_context(node) + + def visit_mermaid(self, node: nodes.Element) -> None: + self.add(f"```mermaid\n{node['code'].strip()}\n```", prefix_eol=2, suffix_eol=2) + raise nodes.SkipNode + + def visit_caption(self, node: nodes.Element) -> None: + self.add(f"**{node.astext()}**", prefix_eol=2, suffix_eol=1) + raise nodes.SkipNode + + # ``abbreviation`` wraps the bare ``*`` that marks keyword-only parameters in signatures. + def visit_abbreviation(self, node: nodes.Element) -> None: + pass + + def depart_abbreviation(self, node: nodes.Element) -> None: + pass + + def visit_title(self, node: nodes.Element) -> None: + # Render navigation captions in bold rather than as section headings. + parent = node.parent + if (isinstance(parent, addnodes.compact_paragraph) and parent.get("toctree")) or ( + isinstance(parent, nodes.compound) and "toctree-wrapper" in parent.get("classes", []) + ): + self.add(f"**{node.astext()}**", prefix_eol=2, suffix_eol=1) + raise nodes.SkipNode + + super().visit_title(node) + + def depart_title(self, node: nodes.Element) -> None: + self._pop_context(node) + + +def _escape_link_text(text: str) -> str: + return text.replace("[", r"\[").replace("]", r"\]") + + +def _inline_markdown(node: nodes.Node, env: BuildEnvironment) -> str: + """Flatten a title or paragraph to one line of Markdown, keeping inline code as code.""" + parts: list[str] = [] + for child in node.children: + if isinstance(child, addnodes.pending_xref) and not child.get("refexplicit"): + parts.append(_xref_title(child, env) or _inline_markdown(child, env)) + elif isinstance(child, nodes.literal): + parts.append(f"`{child.astext()}`") + elif isinstance(child, nodes.Text): + parts.append(child.astext()) + elif isinstance(child, nodes.Element): + parts.append(_inline_markdown(child, env)) + return re.sub(r"\s+", " ", "".join(parts)).strip().translate(_TYPOGRAPHY) + + +def _xref_title(xref: addnodes.pending_xref, env: BuildEnvironment) -> str | None: + """ + Return the title a ``:doc:`` or ``:ref:`` without explicit text resolves to. + + Doctrees read back from the environment are unresolved, so such a reference still holds its + raw target (``operators/llm``) rather than the page or section title the HTML shows. + """ + target = xref.get("reftarget", "") + if xref.get("reftype") == "doc": + refdoc = xref.get("refdoc", "") + docname = ( + target.lstrip("/") + if target.startswith("/") + else posixpath.join(posixpath.dirname(refdoc), target) + ) + title = env.titles.get(posixpath.normpath(docname)) + return _inline_markdown(title, env) if title else None + if xref.get("reftype") == "ref": + domain = cast("StandardDomain", env.get_domain("std")) + label = domain.labels.get(target.lower()) + return label[2] if label else None + return None + + +def _separate_markdown_doctrees(app: Sphinx, config: Config) -> None: + """Give the Markdown build its own doctree cache.""" Review Comment: **[nit]** The same applies here and to `_clean_markdown_doctree` (L273). The PoC's docstrings carried facts that these replacements no longer state: - `_separate_markdown_doctrees`: the new cache sits inside sphinx-llm's temporary `_markdown_build` directory, which is deleted after the copy. So the Markdown build never reuses a doctree. That is the intended behaviour, and it also means this function goes away once the sphinx-llm bug is fixed upstream. - `_clean_markdown_doctree`: `|version|` in a URL is not an RST substitution. `extra_provider_files_with_substitutions` only rewrites the built HTML files, which is why the Markdown build has to do the same on the doctree. These are workarounds for library behaviour, and that is exactly what comments are for. Please restore them. ########## uv.lock: ########## @@ -1,5 +1,5 @@ version = 1 -revision = 5 +revision = 3 Review Comment: **[warning]** The lock file carries about 560 lines of churn that this change doesn't need. The diff is +299/−267: - the `revision` field goes from 5 to 3; - `[options.exclude-newer-package]` is reordered; - resolution markers change on packages unrelated to this PR. When I regenerated it from `main`'s lock with only this PR's `pyproject.toml`, the result was +33/−1: the two new packages and the devel-common `docs` entries. `uv lock --check` passes either way, so this is not a correctness problem. It is a conflict magnet and hides the real change from reviewers. Please reset `uv.lock` to `main` and run `uv lock` again, ideally with a uv recent enough to keep `revision = 5`. ########## devel-common/src/docs/utils/example_source.py: ########## @@ -0,0 +1,43 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from __future__ import annotations + +import subprocess + +from docs.utils.conf_constants import AIRFLOW_REPO_ROOT_PATH + + +def example_source_ref(tag: str) -> str: + """Return HEAD if a remote-tracking ref or tag contains it; otherwise return the release tag.""" Review Comment: **[nit]** The move dropped the reason for the rule. The PoC's docstring explained why the function picks a ref: - `main` docs describe examples that the last release tag doesn't have yet. - The docs publish workflow cherry-picks fixes onto the tag, and that creates a HEAD that doesn't exist on GitHub. The new one-liner only repeats what the code does. In six months, nobody reading this will know why a local-only HEAD falls back to the tag. Please bring the reason back, either in the docstring or as a comment above the `for-each-ref` call. A comment that earns its place explains a nuance the code cannot show, such as a workaround or why the obvious approach is wrong ([PEP 8, *Comments*](https://peps.python.org/pep-0008/#comments)). (see also comments [2] and [4]) ########## devel-common/src/sphinx_exts/redirects.py: ########## @@ -28,67 +28,61 @@ log = logging.getLogger(__name__) -def generate_redirects(app): - """Generate redirects files.""" +def iter_redirects(app): + """Yield source and resolved destination paths for redirects.""" redirect_file_path = os.path.join(app.srcdir, app.config.redirects_file) if not os.path.exists(redirect_file_path): - log.info("Could not found the redirect file: %s", redirect_file_path) + log.info("Could not find the redirect file: %s", redirect_file_path) return in_suffix = next(iter(app.config.source_suffix.keys())) - if not isinstance(app.builder, builders.StandaloneHTMLBuilder): - return - with open(redirect_file_path) as redirects: for line in redirects: - # Skip empty line - if not line.strip(): - continue - - # Skip comments - if line.startswith("#"): + if not line.strip() or line.startswith("#"): continue - # Split line into the original path `from_path` and where the URL should redirect to `to_path` from_path, _, to_path = line.rstrip().partition(" ") log.debug("Redirecting '%s' to '%s'", from_path, to_path) - # in_suffix is often ".rst" from_path = from_path.replace(in_suffix, ".html") to_path = to_path.replace(in_suffix, ".html") - to_path_prefix = f"..{os.path.sep}" * (len(from_path.split(os.path.sep)) - 1) - # The redirect path needs to move back to the root of the apache-airflow docs directory - # or the root of the docs directory altogether for provider distributions. + # Preserve the existing handling of provider redirect paths. Review Comment: **[nit]** "Preserve the existing handling of provider redirect paths" replaces a comment that explained the `..` prefixes: they climb back to the root of the apache-airflow docs, or to the docs root for provider distributions. The new comment talks about the refactor, not the code, so the next reader can't tell why `"providers" in to_path` adds one extra `..`. Please restore the explanation. Dropping the always-true `"../" and` while keeping the behaviour is a good cleanup, and `test_iter_redirects` now pins that behaviour. ########## devel-common/tests/unit/sphinx_exts/test_airflow_llms_txt.py: ########## @@ -0,0 +1,640 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from __future__ import annotations + +import io +from pathlib import Path +from types import SimpleNamespace + +import pytest +from docutils import nodes +from sphinx import addnodes +from sphinx.application import Sphinx +from sphinx.errors import ExtensionError + +from sphinx_exts import airflow_llms_txt + + +def make_app(tmp_path, *, found_docs=()): + source = tmp_path / "source" + output = tmp_path / "output" + source.mkdir() + output.mkdir() + (source / "redirects.txt").write_text("", encoding="utf-8") + + return SimpleNamespace( + srcdir=source, + outdir=output, + builder=SimpleNamespace(name="html"), + env=SimpleNamespace(found_docs=set(found_docs)), + config=SimpleNamespace( + redirects_file="redirects.txt", + source_suffix={".rst": "restructuredtext"}, + llms_txt_enabled=True, + root_doc="index", + ), + ) + + [email protected]( + ("source", "target", "expected"), + [ + pytest.param( + "old.rst", + "new.rst", + "new.html.md", + id="normal", + ), + pytest.param( + "hooks/index.rst", + "concepts.rst", + "../concepts.html.md", + id="nested", + ), + pytest.param( + "old.rst", + "new.rst#section", + "new.html.md#section", + id="anchor", + ), + pytest.param( + "old.rst", + "new.rst?view=full#section", + "new.html.md?view=full#section", + id="query-and-anchor", + ), + pytest.param( + "old.rst", + "../other/new.rst", + "../other/new.html", + id="outside-package", + ), + pytest.param( + "old.rst", + "https://example.org/new.html";, + "https://example.org/new.html";, + id="external", + ), + ], +) +def test_write_markdown_redirects(tmp_path, source, target, expected): + app = make_app(tmp_path) + (app.srcdir / "redirects.txt").write_text(f"{source} {target}\n", encoding="utf-8") + + airflow_llms_txt._write_markdown_redirects(app) + + output = app.outdir / source.replace(".rst", ".html.md") + assert output.read_text(encoding="utf-8") == ( + f"# Page moved\n\nThis page has moved to [the new page](<{expected}>).\n" + ) + + +def test_redirect_preserves_actual_document(tmp_path): + app = make_app(tmp_path, found_docs={"old"}) + (app.srcdir / "redirects.txt").write_text("old.rst new.rst\n", encoding="utf-8") + output = app.outdir / "old.html.md" + output.write_text("# Actual documentation\n", encoding="utf-8") + + airflow_llms_txt._write_markdown_redirects(app) + + assert output.read_text(encoding="utf-8") == "# Actual documentation\n" + + +def test_redirect_updates_existing_stub(tmp_path): + app = make_app(tmp_path) + (app.srcdir / "redirects.txt").write_text("old.rst new.rst\n", encoding="utf-8") + output = app.outdir / "old.html.md" + output.write_text("Old redirect destination", encoding="utf-8") + + airflow_llms_txt._write_markdown_redirects(app) + + assert "(<new.html.md>)" in output.read_text(encoding="utf-8") + assert "Old redirect destination" not in output.read_text(encoding="utf-8") + + +def test_complete_markdown_coverage(tmp_path): + for name in ("index.html", "operators/llm.html"): + page = tmp_path / name + page.parent.mkdir(parents=True, exist_ok=True) + page.write_text("HTML", encoding="utf-8") + page.with_name(page.name + ".md").write_text("# Documentation", encoding="utf-8") + + airflow_llms_txt._check_markdown_coverage(tmp_path) + + +def test_missing_markdown_lists_all_missing_pages(tmp_path): + for name in ("index.html", "operators/llm.html"): + page = tmp_path / name + page.parent.mkdir(parents=True, exist_ok=True) + page.write_text("HTML", encoding="utf-8") + + with pytest.raises(ExtensionError) as exc: + airflow_llms_txt._check_markdown_coverage(tmp_path) + + assert str(exc.value) == ("HTML pages missing Markdown copies:\n index.html\n operators/llm.html") + + [email protected]( + "name", + [ + "genindex.html", + "search.html", + "py-modindex.html", + "_modules/index.html", + "_modules/airflow/providers/example.html", + ], +) +def test_coverage_excludes_special_pages(tmp_path, name): + page = tmp_path / name + page.parent.mkdir(parents=True, exist_ok=True) + page.write_text("HTML", encoding="utf-8") + + airflow_llms_txt._check_markdown_coverage(tmp_path) + + +def test_missing_root_markdown_fails_build(tmp_path): + app = make_app(tmp_path, found_docs={"index"}) + (app.outdir / "index.html").write_text("HTML", encoding="utf-8") + + with pytest.raises(ExtensionError, match="index.html"): + airflow_llms_txt._write_llms_txt(app, None) + + [email protected]( + ("builder", "enabled", "exception"), + [ + pytest.param("html", True, RuntimeError("Build failed"), id="failed-build"), + pytest.param("html", False, None, id="disabled"), + pytest.param( + airflow_llms_txt.MARKDOWN_BUILDER, + True, + None, + id="markdown-build", + ), + ], +) +def test_index_generation_skipped(tmp_path, builder, enabled, exception): + app = make_app(tmp_path) + app.builder.name = builder + app.config.llms_txt_enabled = enabled + (app.outdir / "index.html").write_text("HTML", encoding="utf-8") + + airflow_llms_txt._write_llms_txt(app, exception) + + assert not (app.outdir / "llms.txt").exists() + assert not (app.outdir / "llms-full.txt").exists() + + +def test_indexes_follow_navigation_and_use_versioned_urls(tmp_path): + names = {"index", "guide", "second", "changelog", "commits"} + app = make_app(tmp_path, found_docs=names) + site_url = "https://airflow.apache.org/docs/apache-airflow-providers-common-ai/0.10.0"; + + root = nodes.document("", "") + navigation = addnodes.toctree() + navigation["caption"] = "Guides" + navigation["entries"] = [ + ("Getting started", "guide"), + (None, "second"), + (None, "changelog"), + (None, "commits"), + ] + root += navigation + + doctrees = {"index": root} + for name in names - {"index"}: + tree = nodes.document("", "") + tree += nodes.paragraph("", f"This is the documentation description for {name}.") + doctrees[name] = tree + + app.env.titles = {name: nodes.title("", name.capitalize()) for name in names} + app.env.toctree_includes = {} + app.env.get_doctree = doctrees.__getitem__ + + app.config.llms_txt_site_url = site_url + "/" + app.config.llms_txt_optional_docs = ["changelog"] + app.config.llms_txt_skip_docs = ["commits"] + app.config.project = "Common AI" + app.config.release = "0.10.0" + app.config.llms_txt_description = "Common AI documentation." + app.config.llms_txt_intro = "Install the Common AI provider." + app.config.llms_txt_root_description = "" + + for name in names: + (app.outdir / f"{name}.html").write_text("HTML", encoding="utf-8") + content = f"# {name.capitalize()}\n" + if name == "guide": + content += "[Next](second.html.md)\n" + (app.outdir / f"{name}.html.md").write_text(content, encoding="utf-8") + + airflow_llms_txt._write_llms_txt(app, None) + + index = (app.outdir / "llms.txt").read_text(encoding="utf-8") + full = (app.outdir / "llms-full.txt").read_text(encoding="utf-8") + + assert index.startswith("# Common AI 0.10.0\n") + assert "Install the Common AI provider." in index + assert "## Guides" in index + assert ( + f"- [Getting started]({site_url}/guide.html.md): This is the documentation description for guide." + ) in index + assert index.index("/guide.html.md") < index.index("/second.html.md") + + optional = index.split("## Optional\n", 1)[1] + assert f"- [Changelog]({site_url}/changelog.html.md)" in optional + assert f"[llms-full.txt]({site_url}/llms-full.txt)" in optional + assert "/commits.html.md" not in index + + assert full.index("Source: " + site_url + "/index.html.md") < full.index( + "Source: " + site_url + "/guide.html.md" + ) + assert full.index("Source: " + site_url + "/guide.html.md") < full.index( + "Source: " + site_url + "/second.html.md" + ) + assert f"[Next]({site_url}/second.html.md)" in full + assert "Source: " + site_url + "/changelog.html.md" not in full + assert "Source: " + site_url + "/commits.html.md" not in full + + [email protected]( + ("inside_repo", "source_url", "builder"), + [ + pytest.param( + True, "https://github.com/apache/airflow/blob/abc";, airflow_llms_txt.MARKDOWN_BUILDER, id="linked" + ), + pytest.param( + False, + "https://github.com/apache/airflow/blob/abc";, + airflow_llms_txt.MARKDOWN_BUILDER, + id="outside-repo", + ), + pytest.param(True, "", airflow_llms_txt.MARKDOWN_BUILDER, id="no-source-url"), + pytest.param(True, "https://github.com/apache/airflow/blob/abc";, "html", id="html-unchanged"), + ], +) +def test_example_source_links(tmp_path, monkeypatch, inside_repo, source_url, builder): + monkeypatch.syspath_prepend(str(Path(airflow_llms_txt.__file__).parent)) + + from exampleinclude import ExampleHeader + + repo = tmp_path / "repo" + example = ( + repo / "providers/common/ai/tests/system/example.py" + if inside_repo + else tmp_path / "outside/example.py" + ) + tree = nodes.document("", "") + header = ExampleHeader() + header["filename"] = str(example) + tree += header + + app = SimpleNamespace( + builder=SimpleNamespace(name=builder), + config=SimpleNamespace( + llms_txt_repo_root=str(repo), + llms_txt_source_url=source_url + "/" if source_url else "", + ), + ) + + airflow_llms_txt._link_example_sources(app, tree) + + if builder == "html": + assert tree.children == [header] + elif not inside_repo or not source_url: + assert not tree.children + else: + assert not list(tree.findall(ExampleHeader)) + references = list(tree.findall(nodes.reference)) + assert len(references) == 1 + assert references[0]["refuri"] == (source_url + "/providers/common/ai/tests/system/example.py") + assert references[0].astext() == ("providers/common/ai/tests/system/example.py") + assert tree.astext() == ("Excerpt from providers/common/ai/tests/system/example.py:") + + [email protected]( + ("url", "expected"), + [ + pytest.param( + "https://example.org/docs/|version|/guide", + "https://example.org/docs/0.10.0/guide";, + id="version", + ), + pytest.param( + "https://github.com/apache/airflow/blob/release-tag/example.py";, + "https://github.com/apache/airflow/blob/head-sha/example.py";, + id="github-file", + ), + pytest.param( + "https://github.com/apache/airflow/tree/release-tag/examples";, + "https://github.com/apache/airflow/tree/head-sha/examples";, + id="github-directory", + ), + pytest.param( + "/docs/apache-airflow/stable/index.html", + "https://airflow.apache.org/docs/apache-airflow/stable/index.html";, + id="site-root", + ), + pytest.param( + "//example.org/page", + "//example.org/page", + id="protocol-relative-unchanged", + ), + pytest.param( + "concepts.html.md#section", + "concepts.html.md#section", + id="relative-unchanged", + ), + pytest.param( + "https://example.org/page";, + "https://example.org/page";, + id="external-unchanged", + ), + ], +) +def test_clean_markdown_link_targets(url, expected): + tree = nodes.document("", "") + reference = nodes.reference("", "Link", refuri=url) + tree += reference + + app = SimpleNamespace( + builder=SimpleNamespace(name=airflow_llms_txt.MARKDOWN_BUILDER), + config=SimpleNamespace( + version="0.10.0", + llms_txt_site_url=("https://airflow.apache.org/docs/common-ai/0.10.0/";), + llms_txt_release_source_url=("https://github.com/apache/airflow/blob/release-tag/";), + llms_txt_source_url=("https://github.com/apache/airflow/blob/head-sha/";), + ), + ) + + airflow_llms_txt._clean_markdown_doctree(app, tree, "index") + + assert reference["refuri"] == expected + + +def test_clean_markdown_removes_comments_and_empty_quotes(): + tree = nodes.document("", "") + tree += nodes.comment("", "License header") + tree += nodes.block_quote() + populated_quote = nodes.block_quote() + populated_quote += nodes.paragraph("", "Keep this quotation.") + tree += populated_quote + + app = SimpleNamespace( + builder=SimpleNamespace(name=airflow_llms_txt.MARKDOWN_BUILDER), + config=SimpleNamespace( + llms_txt_site_url="https://airflow.apache.org/";, + llms_txt_release_source_url="", + llms_txt_source_url="", + ), + ) + + airflow_llms_txt._clean_markdown_doctree(app, tree, "index") + + assert not list(tree.findall(nodes.comment)) + assert list(tree.findall(nodes.block_quote)) == [populated_quote] + assert tree.astext() == "Keep this quotation." + + +def test_clean_doctree_leaves_html_unchanged(): + tree = nodes.document("", "") + comment = nodes.comment("", "License header") + reference = nodes.reference("", "Link", refuri="/docs/|version|/") + tree += comment + tree += reference + app = SimpleNamespace(builder=SimpleNamespace(name="html")) + + airflow_llms_txt._clean_markdown_doctree(app, tree, "index") + + assert comment in tree.children + assert reference["refuri"] == "/docs/|version|/" + + [email protected]( + ("markdown", "expected"), + [ + pytest.param( + "[Next](other.html.md#section)", + "[Next](https://example.org/docs/other.html.md#section)", + id="relative-link", + ), + pytest.param( + "[Parent](../index.html.md)", + "[Parent](https://example.org/index.html.md)", + id="parent-link", + ), + pytest.param( + "[Section](#setup)", + "[Section](https://example.org/docs/page.html.md#setup)", + id="same-page-link", + ), + pytest.param( + "[External](https://other.example/page)", + "[External](https://other.example/page)", + id="external-link", + ), + pytest.param( + "`[Example](other.html.md)` [Next](other.html.md)", + "`[Example](other.html.md)` [Next](https://example.org/docs/other.html.md)", + id="inline-code", + ), + pytest.param( + "```python\n[Example](other.html.md)\n```\n[Next](other.html.md)", + "```python\n[Example](other.html.md)\n```\n[Next](https://example.org/docs/other.html.md)", + id="fenced-code", + ), + ], +) +def test_absolute_links(markdown, expected): + assert airflow_llms_txt._absolute_links(markdown, "https://example.org/docs/page.html.md";) == expected + + +def test_navigation_handles_nested_pages_duplicates_and_cycles(): + env = SimpleNamespace( + toctree_includes={ + "index": ["guide", "second", "guide"], + "guide": ["child", "second"], + "child": ["index"], + } + ) + seen = set() + + assert airflow_llms_txt._toctree_docs(env, "index", seen) == [ + ("index", None), + ("guide", None), + ("child", None), + ("second", None), + ] + assert airflow_llms_txt._toctree_docs(env, "guide", seen) == [] + + [email protected]("markdown_build", [True, False]) +def test_markdown_doctree_cache_is_separate(tmp_path, markdown_build): + original = tmp_path / "html-doctrees" + app = SimpleNamespace( + outdir=tmp_path / "markdown-output", + doctreedir=original, + tags=SimpleNamespace(has=lambda tag: markdown_build and tag == "sphinx_llm_markdown"), + ) + + airflow_llms_txt._separate_markdown_doctrees(app, SimpleNamespace()) + + expected = app.outdir / ".doctrees" if markdown_build else original + assert app.doctreedir == expected + + [email protected] +def rendered_markdown(tmp_path, monkeypatch): + source = tmp_path / "source" + output = tmp_path / "output" + source.mkdir() + + # The extension imports redirects and exampleinclude by bare module name. + extension_dir = Path(airflow_llms_txt.__file__).parent + source_root = extension_dir.parent + + (source / "conf.py").write_text( + "import sys\n" + f"sys.path.insert(0, {str(source_root)!r})\n" + f"sys.path.insert(0, {str(extension_dir)!r})\n" + "extensions = [\n" + ' "redirects",\n' + ' "sphinx_exts.airflow_llms_txt",\n' + ' "sphinxcontrib.mermaid",\n' + "]\n" + 'root_doc = "index"\n' + 'redirects_file = "redirects.txt"\n' + "llms_txt_enabled = True\n" + 'llms_txt_suffix_mode = "append"\n' + "llms_txt_build_parallel = False\n", + encoding="utf-8", + ) + (source / "redirects.txt").write_text("", encoding="utf-8") + (source / "index.rst").write_text( + """\ +Rendering +========= + +.. _setup-section: + +Setup +----- + +See `Setup`_. + +.. note:: + + Important configuration advice. + +Following paragraph outside the note. + +.. list-table:: + :header-rows: 1 + + * - Name + - Value + * - Short + - A much longer value + +.. code-block:: python + :caption: Example code + + print("hello") + +.. mermaid:: + + graph TD + A --> B + +.. py:function:: configure(*, model_id) + + Configure a model. + +.. toctree:: + :caption: Further reading + + child +""", + encoding="utf-8", + ) + (source / "child.rst").write_text( + "Child page\n==========\n\nChild documentation.\n", + encoding="utf-8", + ) + + warnings = io.StringIO() + app = Sphinx( + srcdir=str(source), + confdir=str(source), + outdir=str(output), + doctreedir=str(tmp_path / "doctrees"), + buildername="html", + status=io.StringIO(), + warning=warnings, + freshenv=True, + ) + app.build(force_all=True) + + assert app.statuscode == 0, warnings.getvalue() + pages = list(output.rglob("index*.md")) + assert len(pages) == 1 + return pages[0].read_text(encoding="utf-8") + + +def test_rendered_mermaid(rendered_markdown): + assert "```mermaid\n" in rendered_markdown + assert "graph TD\n" in rendered_markdown + assert "A --> B" in rendered_markdown + + +def test_rendered_code_caption_and_sample(rendered_markdown): + assert "**Example code**" in rendered_markdown + assert 'print("hello")' in rendered_markdown + assert rendered_markdown.index("**Example code**") < (rendered_markdown.index('print("hello")')) + + +def test_rendered_keyword_only_marker(rendered_markdown): + signature = next(line for line in rendered_markdown.splitlines() if "configure(" in line) + assert "*" in signature + assert "model_id" in signature + + +def test_rendered_toctree_caption(rendered_markdown): + assert "**Further reading**" in rendered_markdown + assert "[Child page]" in rendered_markdown + assert "# Further reading" not in rendered_markdown + + +def test_rendered_admonition_has_clear_boundaries(rendered_markdown): + lines = rendered_markdown.splitlines() + assert any(line.startswith(">") and "**Note:**" in line for line in lines) + assert any(line.startswith(">") and "Important configuration advice." in line for line in lines) + following = next(line for line in lines if "Following paragraph outside the note." in line) + assert not following.startswith(">") + + +def test_rendered_table_is_compact(rendered_markdown): + table_lines = [line.strip() for line in rendered_markdown.splitlines() if line.strip().startswith("|")] + assert "| Name | Value |" in table_lines + assert "|---|---|" in table_lines + assert "| Short | A much longer value |" in table_lines + + +def test_rendered_same_page_link(rendered_markdown): + assert "[Setup]()" not in rendered_markdown + assert "[Setup]" in rendered_markdown + assert "#setup-section" in rendered_markdown or "#setup" in rendered_markdown Review Comment: **[nit]** This assertion can't tell the two anchors apart. `"#setup" in rendered_markdown` is true whenever `"#setup-section"` is, so the `or` accepts either anchor. It would also pass on any other text that contains `#setup`. Please assert the exact link the translator renders, for example `"[Setup](#setup-section)"` or whatever the build produces. That way the test fails if `_fetch_ref_uri` stops resolving the `refid`. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
