This is an automated email from the ASF dual-hosted git repository. spmallette pushed a commit to branch afd in repository https://gitbox.apache.org/repos/asf/tinkerpop.git
commit ef7ead72c1a41baf765f96966f38718f3138028c Author: Stephen Mallette <[email protected]> AuthorDate: Sat Jul 25 14:21:50 2026 +0000 Keep each GraphSON version whole; split provider step semantics per step Two more splitting refinements driven by content shape: - Add an [llms-keep] section attribute (hidden <!-- llms-keep --> marker) that keeps a section's whole subtree on one page even when it exceeds the size budget, and never descends into it. Applied to each GraphSON version (1.0, 2.0, 3.0) so a version is a single coherent file rather than being fragmented mid-version into arbitrarily-named tail pages (_requestmessage_2, _extended_2 are gone; their content now sits with its version). Versions 2.0/3.0 are intentionally over the 50KB budget as a result. - Mark the provider "Steps" (Gremlin semantics) catalog with [llms-explode] so each step's formal semantics becomes its own page, matching the reference step catalog. 176 extension tests pass; 408 cross-page links resolve; the only pages over 50KB are the deliberately-kept GraphSON versions. Assisted-by: Claude Code:claude-opus-4-8 --- docs/src/dev/io/graphson.asciidoc | 5 ++-- docs/src/dev/provider/gremlin-semantics.asciidoc | 2 +- .../tinkerpop/tinkeradoc/MarkdownConverter.java | 21 ++++++++++++++++ .../tinkerpop/tinkeradoc/MarkdownSplitter.java | 29 ++++++++++++++++++++++ .../tinkeradoc/MarkdownConverterProbeTest.java | 7 ++++++ .../tinkerpop/tinkeradoc/MarkdownSplitterTest.java | 22 ++++++++++++++++ 6 files changed, 83 insertions(+), 3 deletions(-) diff --git a/docs/src/dev/io/graphson.asciidoc b/docs/src/dev/io/graphson.asciidoc index 5194f4b6cd..9bb0623807 100644 --- a/docs/src/dev/io/graphson.asciidoc +++ b/docs/src/dev/io/graphson.asciidoc @@ -101,6 +101,7 @@ and it could be different (or change) from server to server. When building appli mime type is made explicit on requests to avoid breaking changes or unexpected results. [[graphson-1d0]] +[llms-summary="GraphSON 1.0: the original format, with per-type sample encodings for graph structure and request/response messages.",llms-keep=""] == Version 1.0 Version 1.0 of GraphSON was released with TinkerPop 3.0.0. It is referred to by the following mime types: @@ -798,7 +799,7 @@ The following `ResponseMessage` is a typical example of the typical successful r ---- [[graphson-2d0]] -[llms-summary="GraphSON 2.0: the type-embedding format introduced in TinkerPop 3.2.2, with per-type sample encodings for graph structure and request/response messages."] +[llms-summary="GraphSON 2.0: the type-embedding format introduced in TinkerPop 3.2.2, with per-type sample encodings for graph structure and request/response messages.",llms-keep=""] == Version 2.0 Version 2.0 of GraphSON was first introduced on TinkerPop 3.2.2. It was designed to be less tied to @@ -3515,7 +3516,7 @@ The following example is a `ZoneOffset` of three hours, six minutes, and nine se ---- [[graphson-3d0]] -[llms-summary="GraphSON 3.0: the current format (TinkerPop 3.3.0+, application/vnd.gremlin-v3.0+json), with per-type sample encodings for graph structure and request/response messages."] +[llms-summary="GraphSON 3.0: the current format (TinkerPop 3.3.0+, application/vnd.gremlin-v3.0+json), with per-type sample encodings for graph structure and request/response messages.",llms-keep=""] == Version 3.0 Version 3.0 of GraphSON was first introduced on TinkerPop 3.3.0 and is represented by the `application/vnd.gremlin-v3.0+json` diff --git a/docs/src/dev/provider/gremlin-semantics.asciidoc b/docs/src/dev/provider/gremlin-semantics.asciidoc index df560f37b7..367dfee06b 100644 --- a/docs/src/dev/provider/gremlin-semantics.asciidoc +++ b/docs/src/dev/provider/gremlin-semantics.asciidoc @@ -520,7 +520,7 @@ The following table maps the notions proposed above to the various `P` operators link:https://github.com/apache/tinkerpop/tree/x.y.z/gremlin-test/src/main/resources/org/apache/tinkerpop/gremlin/test/features/semantics/Equality.feature[Equality Tests], link:https://github.com/apache/tinkerpop/tree/x.y.z/gremlin-test/src/main/resources/org/apache/tinkerpop/gremlin/test/features/semantics/Comparability.feature[Comparability Tests] -[llms-summary="Per-step semantic definitions: the precise input/output behavior each Gremlin step must exhibit, used to verify provider implementations for correctness."] +[llms-summary="Per-step semantic definitions: the precise input/output behavior each Gremlin step must exhibit, used to verify provider implementations for correctness.",llms-explode=""] == Steps While TinkerPop has a full test suite for validating functionality of Gremlin, tests alone aren't always exhaustive or diff --git a/docs/tinkeradoc-extension/src/main/java/org/apache/tinkerpop/tinkeradoc/MarkdownConverter.java b/docs/tinkeradoc-extension/src/main/java/org/apache/tinkerpop/tinkeradoc/MarkdownConverter.java index 82ca4c6fc5..fa6de1ff9d 100644 --- a/docs/tinkeradoc-extension/src/main/java/org/apache/tinkerpop/tinkeradoc/MarkdownConverter.java +++ b/docs/tinkeradoc-extension/src/main/java/org/apache/tinkerpop/tinkeradoc/MarkdownConverter.java @@ -61,6 +61,13 @@ public class MarkdownConverter extends StringConverter { */ static final String LLMS_EXPLODE_ATTR = "llms-explode"; + /** + * Section attribute ({@code [llms-keep]}) marking a section whose whole subtree must stay on one + * split page even if it exceeds the size budget. Emitted as a hidden {@code <!-- llms-keep -->} + * marker for {@link MarkdownSplitter}; never rendered into the page body. + */ + static final String LLMS_KEEP_ATTR = "llms-keep"; + public MarkdownConverter(final String backend, final Map<String, Object> opts) { super(backend, opts); } @@ -208,10 +215,24 @@ public class MarkdownConverter extends StringConverter { sb.append(hashes).append(' ').append(section.getTitle()).append("\n\n"); appendLlmsSummary(sb, section.getAttribute(LLMS_SUMMARY_ATTR)); appendExplodeMarker(sb, section.getAttribute(LLMS_EXPLODE_ATTR)); + appendKeepMarker(sb, section.getAttribute(LLMS_KEEP_ATTR)); sb.append(section.getContent()); return sb.toString(); } + /** + * Emits a hidden {@code <!-- llms-keep -->} marker when a section carries the {@code [llms-keep]} + * attribute, telling {@link MarkdownSplitter} to keep this section's whole subtree on a single + * page even if it exceeds the size budget (used e.g. to keep each GraphSON version intact). + * Invisible in rendered output. + */ + private static void appendKeepMarker(final StringBuilder sb, final Object keep) { + if (keep == null) return; + final String v = keep.toString().trim(); + if (v.equalsIgnoreCase("false")) return; + sb.append("<!-- llms-keep -->\n\n"); + } + /** * Emits a hidden {@code <!-- llms-explode -->} marker when a section carries the * {@code [llms-explode]} attribute. The marker tells {@link MarkdownSplitter} to give each direct diff --git a/docs/tinkeradoc-extension/src/main/java/org/apache/tinkerpop/tinkeradoc/MarkdownSplitter.java b/docs/tinkeradoc-extension/src/main/java/org/apache/tinkerpop/tinkeradoc/MarkdownSplitter.java index 4e4e460935..7a25d84631 100644 --- a/docs/tinkeradoc-extension/src/main/java/org/apache/tinkerpop/tinkeradoc/MarkdownSplitter.java +++ b/docs/tinkeradoc-extension/src/main/java/org/apache/tinkerpop/tinkeradoc/MarkdownSplitter.java @@ -61,6 +61,8 @@ class MarkdownSplitter { private static final Pattern INTRA_LINK = Pattern.compile("\\]\\(#([^)]+)\\)"); // Marker MarkdownConverter emits from [llms-explode]: split this section's children per-page. private static final String EXPLODE_MARKER = "<!-- llms-explode -->"; + // Marker MarkdownConverter emits from [llms-keep]: keep this section's whole subtree on one page. + private static final String KEEP_MARKER = "<!-- llms-keep -->"; private static final Logger LOG = Logger.getLogger(MarkdownSplitter.class.getName()); @@ -374,6 +376,21 @@ class MarkdownSplitter { cursor.used = packBudget; // force the next sibling onto its own page/flow continue; } + if (isKeep(child)) { + // Keep-whole section (e.g. a GraphSON version): emit the entire subtree as one page, + // never descending, even if it exceeds the budget. Use it as-is when it fits the + // current page's remaining room; otherwise give it its own page. + if (cursor.used + child.byteSize <= packBudget) { + placeWhole(child, cursor.page, anchorToFile); + cursor.used += child.byteSize; + } else { + final PagePlan page = newPage(child, plans); + placeWhole(child, page, anchorToFile); + cursor.page = page; + cursor.used = Math.max(child.byteSize, packBudget); // next sibling starts fresh + } + continue; + } if (cursor.used + child.byteSize <= packBudget) { placeWhole(child, cursor.page, anchorToFile); cursor.used += child.byteSize; @@ -418,6 +435,18 @@ class MarkdownSplitter { return false; } + /** + * Whether a node is marked keep-whole (its own lines contain the {@code <!-- llms-keep -->} + * marker emitted from the {@code [llms-keep]} attribute). Such a node's entire subtree is emitted + * as a single page and is never descended into, even when it exceeds the byte budget. + */ + private static boolean isKeep(final Node node) { + for (final String line : node.lines) { + if (line.trim().equals(KEEP_MARKER)) return true; + } + return false; + } + /** Places a node (and its whole subtree) onto a page, recording all its anchors' home. */ private void placeWhole(final Node node, final PagePlan page, final Map<String, String> anchorToFile) { page.nodes.add(node); diff --git a/docs/tinkeradoc-extension/src/test/java/org/apache/tinkerpop/tinkeradoc/MarkdownConverterProbeTest.java b/docs/tinkeradoc-extension/src/test/java/org/apache/tinkerpop/tinkeradoc/MarkdownConverterProbeTest.java index 1350ab1788..9a4780303d 100644 --- a/docs/tinkeradoc-extension/src/test/java/org/apache/tinkerpop/tinkeradoc/MarkdownConverterProbeTest.java +++ b/docs/tinkeradoc-extension/src/test/java/org/apache/tinkerpop/tinkeradoc/MarkdownConverterProbeTest.java @@ -122,6 +122,13 @@ public class MarkdownConverterProbeTest { assertThat(md, not(containsString("llms-explode"))); } + @Test + public void llmsKeepAttributeBecomesHiddenMarkerNotBodyText() { + final String md = toMarkdown("= T\n\n[llms-keep=\"\"]\n== Version 2.0\n\nVersion body.\n"); + assertThat(md, containsString("<!-- llms-keep -->")); + assertThat(md, containsString("Version body.")); + } + @Test public void emitsExplicitAnchorFromSectionId() { // An explicit AsciiDoc id must surface as an HTML anchor immediately before the heading so diff --git a/docs/tinkeradoc-extension/src/test/java/org/apache/tinkerpop/tinkeradoc/MarkdownSplitterTest.java b/docs/tinkeradoc-extension/src/test/java/org/apache/tinkerpop/tinkeradoc/MarkdownSplitterTest.java index f5fff5a6f2..28e0d60771 100644 --- a/docs/tinkeradoc-extension/src/test/java/org/apache/tinkerpop/tinkeradoc/MarkdownSplitterTest.java +++ b/docs/tinkeradoc-extension/src/test/java/org/apache/tinkerpop/tinkeradoc/MarkdownSplitterTest.java @@ -202,6 +202,28 @@ public class MarkdownSplitterTest { assertThat(cat.getContent(), not(containsString("The fold() step."))); } + @Test + public void keepMarkerHoldsWholeSubtreeOnOnePageEvenOverBudget() { + // A keep-whole section must stay a single page even when its subtree exceeds the budget, + // rather than being descended/fragmented into per-child pages. + final String md = heading("book", 1, "Book") + "\nintro\n\n" + + heading("version-2", 1, "Version 2.0") + "\n<!-- llms-keep -->\n\nintro\n\n" + + heading("edge", 2, "Edge") + "\n" + filler(30_000) + + heading("vertex", 2, "Vertex") + "\n" + filler(30_000); + final List<MarkdownSplitter.Page> pages = new MarkdownSplitter(50_000).split(md, "index.md"); + final List<String> names = fileNames(pages); + // The whole Version 2.0 subtree is one page; its children did NOT become separate pages. + assertThat(names, hasItem("version-2.md")); + assertThat(names, not(hasItem("edge.md"))); + assertThat(names, not(hasItem("vertex.md"))); + final MarkdownSplitter.Page v2 = pages.stream() + .filter(p -> p.getFileName().equals("version-2.md")).findFirst().orElseThrow(AssertionError::new); + // Contains both children on the one page, and is (intentionally) over the budget. + assertThat(v2.getContent(), containsString("## Edge")); + assertThat(v2.getContent(), containsString("## Vertex")); + assertThat(v2.getContent().length() > 50_000, is(true)); + } + @Test public void hashCommentsInsideCodeFenceAreNotTreatedAsHeadings() { // A code fence containing shell/properties comment lines that begin with '#' must NOT be
