This is an automated email from the ASF dual-hosted git repository. spmallette pushed a commit to branch afd in repository https://gitbox.apache.org/repos/asf/tinkerpop.git
commit e3da9ebfa0dca63c3c38401396635be3d929f024 Author: Stephen Mallette <[email protected]> AuthorDate: Sat Jul 25 15:22:44 2026 +0000 Make Markdown splitting purely summary-driven with a size lint Replace the size-driven splitter (and its llms-explode / llms-keep overrides) with a single rule: a section becomes its own Markdown page if and only if it carries an llms-summary. Unsummarized sections attach to their nearest summarized ancestor's page. This makes page boundaries deterministic and author-controlled, keeps every page described in llms.txt by construction, and deletes the size-packing / descend / explode / keep machinery. Size is now an author concern surfaced by a lint rather than enforced by splitting: MarkdownSplitter reports any page over the 50KB budget, and bin/process-docs.sh runs it with --strict so the docs build fails on an over-budget page. A section may opt out with allow-oversize="true" (emitted as a hidden marker) when it is intentionally kept whole. Curation to make the first summary-driven build clean: - explode/keep attributes converted: traversal + provider step catalogs now split per-step purely because each step has a summary; GraphSON versions stay whole because their per-type children have none; oversized-on-purpose pages (GraphSON 2.0, the 3.2.x release notes, the committer guide) get allow-oversize="true". - add llms-summary to the 31 provider step-semantics sections, the 15 traversal recipes, and the reference Introduction and Gremlin Compilers sections. Result: 246 pages, all described in llms.txt (40KB); the only over-budget pages are the three explicitly flagged allow-oversize; 412 cross-page links resolve; 173 extension tests pass. Assisted-by: Claude Code:claude-opus-4-8 --- bin/process-docs.sh | 13 +- docs/src/dev/developer/for-committers.asciidoc | 2 +- docs/src/dev/io/graphson.asciidoc | 6 +- docs/src/dev/provider/gremlin-semantics.asciidoc | 31 +- docs/src/recipes/between-vertices.asciidoc | 1 + docs/src/recipes/centrality.asciidoc | 1 + docs/src/recipes/collections.asciidoc | 1 + docs/src/recipes/cycle-detection.asciidoc | 1 + docs/src/recipes/duplicate-edge.asciidoc | 1 + docs/src/recipes/duplicate-vertex.asciidoc | 1 + docs/src/recipes/edge-move.asciidoc | 1 + docs/src/recipes/element-existence.asciidoc | 1 + docs/src/recipes/if-then-based-grouping.asciidoc | 1 + docs/src/recipes/looping.asciidoc | 1 + docs/src/recipes/olap-spark-yarn.asciidoc | 1 + .../recipes/operating-on-dropped-elements.asciidoc | 1 + docs/src/recipes/pagination.asciidoc | 1 + docs/src/recipes/shortest-path.asciidoc | 1 + docs/src/recipes/traversal-induced-values.asciidoc | 1 + docs/src/reference/compilers.asciidoc | 1 + docs/src/reference/intro.asciidoc | 1 + docs/src/reference/the-traversal.asciidoc | 2 +- docs/src/upgrade/release-3.2.x-incubating.asciidoc | 2 +- .../tinkerpop/tinkeradoc/MarkdownConverter.java | 52 +-- .../tinkerpop/tinkeradoc/MarkdownSplitter.java | 347 ++++++++------------- .../tinkeradoc/MarkdownConverterProbeTest.java | 28 +- .../tinkerpop/tinkeradoc/MarkdownSplitterTest.java | 266 ++++++---------- 27 files changed, 315 insertions(+), 451 deletions(-) diff --git a/bin/process-docs.sh b/bin/process-docs.sh index 4bfa1bc863..efeea29986 100755 --- a/bin/process-docs.sh +++ b/bin/process-docs.sh @@ -101,8 +101,10 @@ port_open() { # Markdown split (agentdocsspec.com) # --------------------------------------------------------------------------- # After the Markdown mirror is rendered under target/docs/markdown/, split each rendered book into -# agent-sized (<=50KB) pages and rewrite intra-book links. Runs MarkdownSplitter from the -# tinkeradoc-extension classes (built by the asciidoc profile) with asciidoctorj on the classpath. +# pages driven by llms-summary markers and rewrite intra-book links. Runs MarkdownSplitter from the +# tinkeradoc-extension classes (built by the asciidoc profile). The split is summary-driven: a +# section is a page iff it has an llms-summary. --strict fails the build on any page that exceeds +# the size budget and is not marked allow-oversize, so uncurated/oversized content is surfaced. split_markdown() { local md_root="target/docs/markdown" [ -d "${md_root}" ] || return 0 @@ -111,14 +113,13 @@ split_markdown() { echo "WARNING: ${ext_classes} not found; skipping Markdown split." return 0 fi - # asciidoctorj jars are only needed for the extension build, not for the splitter (pure Java), - # so the extension classes alone suffice on the classpath. + # The splitter is pure Java, so the extension classes alone suffice on the classpath. local books books=$(find "${md_root}" -name index.md 2>/dev/null) [ -z "${books}" ] && return 0 - echo "Splitting Markdown books into agent-sized pages..." + echo "Splitting Markdown books into agent-sized pages (summary-driven)..." # shellcheck disable=SC2086 - java -cp "${ext_classes}" org.apache.tinkerpop.tinkeradoc.MarkdownSplitter ${books} + java -cp "${ext_classes}" org.apache.tinkerpop.tinkeradoc.MarkdownSplitter --strict ${books} # Generate the llms.txt discovery index over the split pages (agentdocsspec.com). echo "Generating llms.txt discovery index..." diff --git a/docs/src/dev/developer/for-committers.asciidoc b/docs/src/dev/developer/for-committers.asciidoc index 172c06b965..c4a5af9e4e 100644 --- a/docs/src/dev/developer/for-committers.asciidoc +++ b/docs/src/dev/developer/for-committers.asciidoc @@ -16,7 +16,7 @@ KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. //// -[llms-summary="Guidelines for TinkerPop committers: the Review-then-Commit (RTC) process, commit conventions, branch and merge practices, and committer responsibilities."] +[llms-summary="Guidelines for TinkerPop committers: the Review-then-Commit (RTC) process, commit conventions, branch and merge practices, and committer responsibilities.",allow-oversize="true"] = For Committers image::business-gremlin.png[width=400] diff --git a/docs/src/dev/io/graphson.asciidoc b/docs/src/dev/io/graphson.asciidoc index 9bb0623807..2e5a165bf4 100644 --- a/docs/src/dev/io/graphson.asciidoc +++ b/docs/src/dev/io/graphson.asciidoc @@ -101,7 +101,7 @@ and it could be different (or change) from server to server. When building appli mime type is made explicit on requests to avoid breaking changes or unexpected results. [[graphson-1d0]] -[llms-summary="GraphSON 1.0: the original format, with per-type sample encodings for graph structure and request/response messages.",llms-keep=""] +[llms-summary="GraphSON 1.0: the original format, with per-type sample encodings for graph structure and request/response messages.",allow-oversize="true"] == Version 1.0 Version 1.0 of GraphSON was released with TinkerPop 3.0.0. It is referred to by the following mime types: @@ -799,7 +799,7 @@ The following `ResponseMessage` is a typical example of the typical successful r ---- [[graphson-2d0]] -[llms-summary="GraphSON 2.0: the type-embedding format introduced in TinkerPop 3.2.2, with per-type sample encodings for graph structure and request/response messages.",llms-keep=""] +[llms-summary="GraphSON 2.0: the type-embedding format introduced in TinkerPop 3.2.2, with per-type sample encodings for graph structure and request/response messages.",allow-oversize="true"] == Version 2.0 Version 2.0 of GraphSON was first introduced on TinkerPop 3.2.2. It was designed to be less tied to @@ -3516,7 +3516,7 @@ The following example is a `ZoneOffset` of three hours, six minutes, and nine se ---- [[graphson-3d0]] -[llms-summary="GraphSON 3.0: the current format (TinkerPop 3.3.0+, application/vnd.gremlin-v3.0+json), with per-type sample encodings for graph structure and request/response messages.",llms-keep=""] +[llms-summary="GraphSON 3.0: the current format (TinkerPop 3.3.0+, application/vnd.gremlin-v3.0+json), with per-type sample encodings for graph structure and request/response messages.",allow-oversize="true"] == Version 3.0 Version 3.0 of GraphSON was first introduced on TinkerPop 3.3.0 and is represented by the `application/vnd.gremlin-v3.0+json` diff --git a/docs/src/dev/provider/gremlin-semantics.asciidoc b/docs/src/dev/provider/gremlin-semantics.asciidoc index 367dfee06b..30dac43740 100644 --- a/docs/src/dev/provider/gremlin-semantics.asciidoc +++ b/docs/src/dev/provider/gremlin-semantics.asciidoc @@ -520,7 +520,7 @@ The following table maps the notions proposed above to the various `P` operators link:https://github.com/apache/tinkerpop/tree/x.y.z/gremlin-test/src/main/resources/org/apache/tinkerpop/gremlin/test/features/semantics/Equality.feature[Equality Tests], link:https://github.com/apache/tinkerpop/tree/x.y.z/gremlin-test/src/main/resources/org/apache/tinkerpop/gremlin/test/features/semantics/Comparability.feature[Comparability Tests] -[llms-summary="Per-step semantic definitions: the precise input/output behavior each Gremlin step must exhibit, used to verify provider implementations for correctness.",llms-explode=""] +[llms-summary="Per-step semantic definitions: the precise input/output behavior each Gremlin step must exhibit, used to verify provider implementations for correctness.""] == Steps While TinkerPop has a full test suite for validating functionality of Gremlin, tests alone aren't always exhaustive or @@ -528,6 +528,7 @@ fully demonstrative of Gremlin step semantics. It is also hard to simply read th step is meant to behave. This section discusses the semantics for individual steps to help users and providers understand implementation expectations. +[llms-summary="The formal semantics of the all() step: filters array data from the Traversal Stream if all of the array's items match the supplied predicate."] [[all-step]] === all() @@ -561,6 +562,7 @@ will be filtered out of the Traversal Stream. See: link:https://github.com/apache/tinkerpop/tree/x.y.z/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/filter/AllStep.java[source], link:https://tinkerpop.apache.org/docs/x.y.z/reference/#all-step[reference] +[llms-summary="The formal semantics of the any() step: filters array data from the Traversal Stream if any of the array's items match the supplied predicate."] [[any-step]] === any() @@ -594,6 +596,7 @@ filtered out of the Traversal Stream. See: link:https://github.com/apache/tinkerpop/tree/x.y.z/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/filter/AnyStep.java[source], link:https://tinkerpop.apache.org/docs/x.y.z/reference/#any-step[reference] +[llms-summary="The formal semantics of the asDate() step: parse the value of incoming traverser as date."] [[asDate-step]] === asDate() @@ -620,6 +623,7 @@ Incoming date remains unchanged. See: link:https://github.com/apache/tinkerpop/tree/x.y.z/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/AsDateStep.java[source], link:https://tinkerpop.apache.org/docs/x.y.z/reference/#asDate-step[reference] +[llms-summary="The formal semantics of the asString() step: returns the value of incoming traverser as strings, or if Scope.local is specified, returns each element inside incoming list traverser as string."] [[asString-step]] === asString() @@ -651,6 +655,7 @@ See: link:https://github.com/apache/tinkerpop/tree/x.y.z/gremlin-core/src/main/j link:https://github.com/apache/tinkerpop/tree/x.y.z/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/AsStringLocalStep.java[source (local)], link:https://tinkerpop.apache.org/docs/x.y.z/reference/#asString-step[reference] +[llms-summary="The formal semantics of the call() step: provides support for provider-specific service calls."] [[call-step]] === call() @@ -773,6 +778,7 @@ See: link:https://github.com/apache/tinkerpop/tree/x.y.z/gremlin-core/src/main/j link:https://tinkerpop.apache.org/docs/x.y.z/reference/#combine-step[reference], link:https://tinkerpop.apache.org/docs/x.y.z/reference/#merge-step[merge() reference] +[llms-summary="The formal semantics of the concat() step: concatenates the incoming String traverser with the input String arguments, and return the joined String."] [[concat-step]] === concat() @@ -805,6 +811,7 @@ concatenated, the `null` value will be propagated and returned. See: link:https://github.com/apache/tinkerpop/tree/x.y.z/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/ConcatStep.java[source], link:https://tinkerpop.apache.org/docs/x.y.z/reference/#concat-step[reference] +[llms-summary="The formal semantics of the dateAdd() step: increase value of input Date."] [[dateAdd-step]] === dateAdd() @@ -830,6 +837,7 @@ link:https://tinkerpop.apache.org/docs/x.y.z/reference/#concat-step[reference] See: link:https://github.com/apache/tinkerpop/tree/x.y.z/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/DateAddStep.java[source], link:https://tinkerpop.apache.org/docs/x.y.z/reference/#dateAdd-step[reference] +[llms-summary="The formal semantics of the dateDiff() step: returns the difference between two Dates in epoch time."] [[dateDiff-step]] === dateDiff() @@ -858,6 +866,7 @@ If argument resolves as `null` then incoming date will not be changed. See: link:https://github.com/apache/tinkerpop/tree/x.y.z/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/DateDiffStep.java[source], link:https://tinkerpop.apache.org/docs/x.y.z/reference/#dateDiff-step[reference] +[llms-summary="The formal semantics of the dedup() step: removes repeatedly seen results from the Traversal Stream."] [[dedup-step]] === dedup() @@ -926,6 +935,7 @@ See: link:https://github.com/apache/tinkerpop/tree/x.y.z/gremlin-core/src/main/j link:https://github.com/apache/tinkerpop/tree/x.y.z/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/DedupLocalStep.java[source (local)], link:https://tinkerpop.apache.org/docs/x.y.z/reference/#dedup-step[reference] +[llms-summary="The formal semantics of the difference() step: adds the difference of two lists to the Traversal Stream."] [[difference-step]] === difference() @@ -961,6 +971,7 @@ applies to list types which means that non-iterable types (including null) will See: link:https://github.com/apache/tinkerpop/tree/x.y.z/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/DifferenceStep.java[source], link:https://tinkerpop.apache.org/docs/x.y.z/reference/#difference-step[reference] +[llms-summary="The formal semantics of the disjunct() step: adds the disjunct set to the Traversal Stream."] [[disjunct-step]] === disjunct() @@ -995,6 +1006,7 @@ types which means that non-iterable types (including null) will cause exceptions See: link:https://github.com/apache/tinkerpop/tree/x.y.z/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/DisjunctStep.java[source], link:https://tinkerpop.apache.org/docs/x.y.z/reference/#disjunct-step[reference] +[llms-summary="The formal semantics of the element() step: traverse from Property to its Element."] [[element-step]] === element() @@ -1016,6 +1028,7 @@ None None +[llms-summary="The formal semantics of the format() step: a mid-traversal step which will handle result formatting to string values."] [[format-step]] === format() @@ -1048,6 +1061,7 @@ None See: link:https://github.com/apache/tinkerpop/tree/x.y.z/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/FormatStep.java[source], link:https://tinkerpop.apache.org/docs/x.y.z/reference/#format-step[reference] +[llms-summary="The formal semantics of the length() step: returns the length of the incoming string or list, if Scope.local is specified, returns the length of each string elements inside incoming list traverser."] [[length-step]] === length() @@ -1078,6 +1092,7 @@ See: link:https://github.com/apache/tinkerpop/tree/x.y.z/gremlin-core/src/main/j link:https://github.com/apache/tinkerpop/tree/x.y.z/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/LengthLocalStep.java[source (local)], link:https://tinkerpop.apache.org/docs/x.y.z/reference/#length-step[reference] +[llms-summary="The formal semantics of the intersect() step: adds the intersection to the Traversal Stream."] [[intersect-step]] === intersect() @@ -1113,6 +1128,7 @@ types which means that non-iterable types (including null) will cause exceptions See: link:https://github.com/apache/tinkerpop/tree/x.y.z/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/IntersectStep.java[source], link:https://tinkerpop.apache.org/docs/x.y.z/reference/#intersect-step[reference] +[llms-summary="The formal semantics of the conjoin() step: joins every element in a list together into a String."] [[conjoin-step]] === conjoin() @@ -1148,6 +1164,7 @@ non-iterable types (including `null`) will cause exceptions to be thrown. See: link:https://github.com/apache/tinkerpop/tree/x.y.z/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/ConjoinStep.java[source], link:https://tinkerpop.apache.org/docs/x.y.z/reference/#conjoin-step[reference] +[llms-summary="The formal semantics of the lTrim() step: returns a string with leading whitespace removed."] [[lTrim-step]] === lTrim() @@ -1177,6 +1194,7 @@ See: link:https://github.com/apache/tinkerpop/tree/x.y.z/gremlin-core/src/main/j link:https://github.com/apache/tinkerpop/tree/x.y.z/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/LTrimLocalStep.java[source (local)], link:https://tinkerpop.apache.org/docs/x.y.z/reference/#lTrim-step[reference] +[llms-summary="The formal semantics of the merge() step: adds the union of two sets (or two maps) to the Traversal Stream."] [[merge-step]] === merge() @@ -1213,6 +1231,7 @@ to be thrown. See: link:https://github.com/apache/tinkerpop/tree/x.y.z/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/MergeStep.java[source], link:https://tinkerpop.apache.org/docs/x.y.z/reference/#merge-step[reference] +[llms-summary="The formal semantics of the mergeE() step: provides upsert-like functionality for edges."] [[merge-e-step]] === mergeE() @@ -1293,6 +1312,7 @@ be set to `Merge.inV`. Other combinations are not allowed and will result in exc See: link:https://github.com/apache/tinkerpop/tree/x.y.z/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/MergeEdgeStep.java[source], link:https://tinkerpop.apache.org/docs/x.y.z/reference/#mergee-step[reference] +[llms-summary="The formal semantics of the mergeV() step: provides upsert-like functionality for vertices."] [[merge-v-step]] === mergeV() @@ -1359,6 +1379,7 @@ resolve to a `Map`. See: link:https://github.com/apache/tinkerpop/tree/x.y.z/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/MergeVertexStep.java[source], link:https://tinkerpop.apache.org/docs/x.y.z/reference/#mergev-step[reference] +[llms-summary="The formal semantics of the product() step: adds the cartesian product to the Traversal Stream."] [[product-step]] === product() @@ -1393,6 +1414,7 @@ applies to list types which means that non-iterable types (including null) will See: link:https://github.com/apache/tinkerpop/tree/x.y.z/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/ProductStep.java[source], link:https://tinkerpop.apache.org/docs/x.y.z/reference/#product-step[reference] +[llms-summary="The formal semantics of the repeat() step: iteratively applies a traversal (the \"loop body\") to each incoming traverser until a stopping condition is met."] [[repeat-step]] === repeat() @@ -1466,6 +1488,7 @@ See: link:https://github.com/apache/tinkerpop/tree/x.y.z/gremlin-core/src/main/j link:https://tinkerpop.apache.org/docs/x.y.z/reference/#repeat-step[reference], link:https://github.com/apache/tinkerpop/tree/x.y.z/gremlin-test/src/main/resources/org/apache/tinkerpop/gremlin/test/features/branch/Repeat.feature[tests] +[llms-summary="The formal semantics of the replace() step: returns a string with the specified characters in the original string replaced with the new characters."] [[replace-step]] === replace() @@ -1499,6 +1522,7 @@ See: link:https://github.com/apache/tinkerpop/tree/x.y.z/gremlin-core/src/main/j link:https://github.com/apache/tinkerpop/tree/x.y.z/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/ReplaceLocalStep.java[source (local)], link:https://tinkerpop.apache.org/docs/x.y.z/reference/#replace-step[reference] +[llms-summary="The formal semantics of the reverse() step: returns the reverse of the incoming traverser"] [[reverse-step]] === reverse() @@ -1527,6 +1551,7 @@ order are returned. All other types (including null) are not processed and are r See: link:https://github.com/apache/tinkerpop/tree/x.y.z/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/ReverseStep.java[source], link:https://tinkerpop.apache.org/docs/x.y.z/reference/#reverse-step[reference] +[llms-summary="The formal semantics of the rTrim() step: returns a string with trailing whitespace removed."] [[rTrim-step]] === rTrim() @@ -1556,6 +1581,7 @@ See: link:https://github.com/apache/tinkerpop/tree/x.y.z/gremlin-core/src/main/j link:https://github.com/apache/tinkerpop/tree/x.y.z/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/RTrimLocalStep.java[source (local)], link:https://tinkerpop.apache.org/docs/x.y.z/reference/#rTrim-step[reference] +[llms-summary="The formal semantics of the split() step: returns a list of strings created by splitting the incoming string traverser around the matches of the given separator."] [[split-step]] === split() @@ -1621,6 +1647,7 @@ See: link:https://github.com/apache/tinkerpop/tree/x.y.z/gremlin-core/src/main/j link:https://github.com/apache/tinkerpop/tree/x.y.z/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/SubstringLocalStep.java[source (local)], link:https://tinkerpop.apache.org/docs/x.y.z/reference/#substring-step[reference] +[llms-summary="The formal semantics of the toLower() step: returns the lowercase representation of incoming string traverser, or if Scope.local is specified, returns the lowercase representation of each string elemen"] [[toLower-step]] === toLower() @@ -1651,6 +1678,7 @@ See: link:https://github.com/apache/tinkerpop/tree/x.y.z/gremlin-core/src/main/j link:https://github.com/apache/tinkerpop/tree/x.y.z/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/ToLowerLocalStep.java[source (local)], link:https://tinkerpop.apache.org/docs/x.y.z/reference/#toLower-step[reference] +[llms-summary="The formal semantics of the toUpper() step: returns the uppercase representation of incoming string traverser, or if Scope.local is specified, returns the uppercase representation of each string elemen"] [[toUpper-step]] === toUpper() @@ -1681,6 +1709,7 @@ See: link:https://github.com/apache/tinkerpop/tree/x.y.z/gremlin-core/src/main/j link:https://github.com/apache/tinkerpop/tree/x.y.z/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/ToUpperLocalStep.java[source (local)], link:https://tinkerpop.apache.org/docs/x.y.z/reference/#toUpper-step[reference] +[llms-summary="The formal semantics of the trim() step: returns a string with leading and trailing whitespace removed."] [[trim-step]] === trim() diff --git a/docs/src/recipes/between-vertices.asciidoc b/docs/src/recipes/between-vertices.asciidoc index 4d7e413db6..a4dbde30f8 100644 --- a/docs/src/recipes/between-vertices.asciidoc +++ b/docs/src/recipes/between-vertices.asciidoc @@ -16,6 +16,7 @@ KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. //// +[llms-summary="Recipe: It is quite common to have a situation where there are two particular vertices of a graph and a need to execute some traversal on the paths found between them."] [[between-vertices]] == Between Vertices diff --git a/docs/src/recipes/centrality.asciidoc b/docs/src/recipes/centrality.asciidoc index 53422f8257..e5b99fbc40 100644 --- a/docs/src/recipes/centrality.asciidoc +++ b/docs/src/recipes/centrality.asciidoc @@ -16,6 +16,7 @@ KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. //// +[llms-summary="Recipe: There are many measures of centrality which are meant to help identify the most important vertices in a graph."] [[centrality]] == Centrality diff --git a/docs/src/recipes/collections.asciidoc b/docs/src/recipes/collections.asciidoc index 34d86c5bde..1dab723f74 100644 --- a/docs/src/recipes/collections.asciidoc +++ b/docs/src/recipes/collections.asciidoc @@ -16,6 +16,7 @@ KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. //// +[llms-summary="Recipe: A recipe for collections in Gremlin."] [[collections]] == Collections diff --git a/docs/src/recipes/cycle-detection.asciidoc b/docs/src/recipes/cycle-detection.asciidoc index 69eb0df813..d15fcea169 100644 --- a/docs/src/recipes/cycle-detection.asciidoc +++ b/docs/src/recipes/cycle-detection.asciidoc @@ -16,6 +16,7 @@ KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. //// +[llms-summary="Recipe: A cycle occurs in a graph where a path loops back on itself to the originating vertex."] [[cycle-detection]] == Cycle Detection diff --git a/docs/src/recipes/duplicate-edge.asciidoc b/docs/src/recipes/duplicate-edge.asciidoc index b7196a187f..be31286047 100644 --- a/docs/src/recipes/duplicate-edge.asciidoc +++ b/docs/src/recipes/duplicate-edge.asciidoc @@ -16,6 +16,7 @@ KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. //// +[llms-summary="Recipe: Whether part of a graph maintenance process or for some other analysis need, it is sometimes necessary to detect if there is more than one edge between two vert"] [[duplicate-edge]] == Duplicate Edge Detection diff --git a/docs/src/recipes/duplicate-vertex.asciidoc b/docs/src/recipes/duplicate-vertex.asciidoc index 4fdfc724f3..143b46f1be 100644 --- a/docs/src/recipes/duplicate-vertex.asciidoc +++ b/docs/src/recipes/duplicate-vertex.asciidoc @@ -16,6 +16,7 @@ KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. //// +[llms-summary="Recipe: The pattern for finding duplicate vertices is quite similar to the pattern defined in the Duplicate Edge section."] [[duplicate-vertex]] == Duplicate Vertex Detection diff --git a/docs/src/recipes/edge-move.asciidoc b/docs/src/recipes/edge-move.asciidoc index e2ffd3a085..25a416d030 100644 --- a/docs/src/recipes/edge-move.asciidoc +++ b/docs/src/recipes/edge-move.asciidoc @@ -16,6 +16,7 @@ KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. //// +[llms-summary="Recipe: A recipe for moving an edge in Gremlin."] [[edge-move]] == Moving an Edge diff --git a/docs/src/recipes/element-existence.asciidoc b/docs/src/recipes/element-existence.asciidoc index a6df9e6693..137db270cf 100644 --- a/docs/src/recipes/element-existence.asciidoc +++ b/docs/src/recipes/element-existence.asciidoc @@ -16,6 +16,7 @@ KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. //// +[llms-summary="Recipe: Checking for whether or not a graph element is present in the graph is simple:"] [[element-existence]] == Element Existence diff --git a/docs/src/recipes/if-then-based-grouping.asciidoc b/docs/src/recipes/if-then-based-grouping.asciidoc index 3b483802f2..5d88b9a6ab 100644 --- a/docs/src/recipes/if-then-based-grouping.asciidoc +++ b/docs/src/recipes/if-then-based-grouping.asciidoc @@ -16,6 +16,7 @@ KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. //// +[llms-summary="Recipe: Consider the following traversal over the \"modern\" toy graph:"] [[if-then-based-grouping]] == If-Then Based Grouping diff --git a/docs/src/recipes/looping.asciidoc b/docs/src/recipes/looping.asciidoc index 79da4136d1..2f9042e69a 100644 --- a/docs/src/recipes/looping.asciidoc +++ b/docs/src/recipes/looping.asciidoc @@ -16,6 +16,7 @@ KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. //// +[llms-summary="Recipe: One common use case when working with Gremlin is to perform complex looping statements using the repeat() step."] [[looping]] == Looping One common use case when working with Gremlin is to perform complex looping statements using the `repeat()` step. While many of the common patterns for looping within traversals are discussed in the documentation there are several more complex patterns that include additional steps which are also commonly used. This section attempts to demonstrate how to use some of these more complex measurements. diff --git a/docs/src/recipes/olap-spark-yarn.asciidoc b/docs/src/recipes/olap-spark-yarn.asciidoc index 12ea3c77d0..edb0b33a0c 100644 --- a/docs/src/recipes/olap-spark-yarn.asciidoc +++ b/docs/src/recipes/olap-spark-yarn.asciidoc @@ -16,6 +16,7 @@ KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. //// +[llms-summary="Recipe: TinkerPop's combination of SparkGraphComputer and HadoopGraph allows for running distributed, analytical graph queries (OLAP) on a computer cluster."] [[olap-spark-yarn]] == OLAP traversals with Spark on YARN diff --git a/docs/src/recipes/operating-on-dropped-elements.asciidoc b/docs/src/recipes/operating-on-dropped-elements.asciidoc index ca549a89c8..98b6dff389 100644 --- a/docs/src/recipes/operating-on-dropped-elements.asciidoc +++ b/docs/src/recipes/operating-on-dropped-elements.asciidoc @@ -16,6 +16,7 @@ KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. //// +[llms-summary="Recipe: One common scenario that happens when dropping elements using a traversal is the desire to perform some sort of operation on the elements being removed."] [[operating-on-dropped-elements]] == Operating on Dropped Elements diff --git a/docs/src/recipes/pagination.asciidoc b/docs/src/recipes/pagination.asciidoc index af5d0db989..a331bbf7b3 100644 --- a/docs/src/recipes/pagination.asciidoc +++ b/docs/src/recipes/pagination.asciidoc @@ -16,6 +16,7 @@ KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. //// +[llms-summary="Recipe: In most database applications, it is oftentimes desirable to return discrete blocks of data for a query rather than all of the data that the total results would"] [[pagination]] == Pagination diff --git a/docs/src/recipes/shortest-path.asciidoc b/docs/src/recipes/shortest-path.asciidoc index 4dc87e60d8..89a0577e9d 100644 --- a/docs/src/recipes/shortest-path.asciidoc +++ b/docs/src/recipes/shortest-path.asciidoc @@ -16,6 +16,7 @@ KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. //// +[llms-summary="Recipe: A recipe for shortest path in Gremlin."] [[shortest-path]] == Shortest Path diff --git a/docs/src/recipes/traversal-induced-values.asciidoc b/docs/src/recipes/traversal-induced-values.asciidoc index 162651996f..ad7637826e 100644 --- a/docs/src/recipes/traversal-induced-values.asciidoc +++ b/docs/src/recipes/traversal-induced-values.asciidoc @@ -16,6 +16,7 @@ KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. //// +[llms-summary="Recipe: The parameters of a Traversal can be known ahead of time as constants or might otherwise be passed in as dynamic arguments."] [[traversal-induced-values]] == Traversal Induced Values diff --git a/docs/src/reference/compilers.asciidoc b/docs/src/reference/compilers.asciidoc index 07b1b0a1f2..8fc5a8d40b 100644 --- a/docs/src/reference/compilers.asciidoc +++ b/docs/src/reference/compilers.asciidoc @@ -17,6 +17,7 @@ specific language governing permissions and limitations under the License. //// [[compilers]] +[llms-summary="Gremlin compilers translate other query languages into Gremlin bytecode; covers SPARQL-Gremlin for querying graphs with SPARQL."] = Gremlin Compilers There are many languages built to query data. SQL is typically used to query relational data. There is SPARQL for RDF diff --git a/docs/src/reference/intro.asciidoc b/docs/src/reference/intro.asciidoc index fda8423551..43c127f018 100644 --- a/docs/src/reference/intro.asciidoc +++ b/docs/src/reference/intro.asciidoc @@ -17,6 +17,7 @@ specific language governing permissions and limitations under the License. //// [[intro]] +[llms-summary="Introduction to TinkerPop and Gremlin: graph computing concepts (structure vs. process), the ways to connect to Gremlin (embedded, Gremlin Server, remote providers), and how to stay implementation-agnostic."] = Introduction Welcome to the Reference Documentation for Apache TinkerPop™ - the backbone for all details on how to work with diff --git a/docs/src/reference/the-traversal.asciidoc b/docs/src/reference/the-traversal.asciidoc index 6d12ab989d..c9ef3c24df 100644 --- a/docs/src/reference/the-traversal.asciidoc +++ b/docs/src/reference/the-traversal.asciidoc @@ -390,7 +390,7 @@ Spawn steps, which actually yield a traversal, typically match the names of exis * `V()` - Reads vertices from the graph to start the traversal (<<graph-step, example>>). [[graph-traversal-steps]] -[llms-summary="Reference catalog of the individual Gremlin steps (map, filter, sideEffect, branch, and more) with syntax and examples for each.",llms-explode=""] +[llms-summary="Reference catalog of the individual Gremlin steps (map, filter, sideEffect, branch, and more) with syntax and examples for each.""] == Graph Traversal Steps Gremlin steps are chained together to produce the actual traversal and are triggered by way of <<start-steps,start steps>> diff --git a/docs/src/upgrade/release-3.2.x-incubating.asciidoc b/docs/src/upgrade/release-3.2.x-incubating.asciidoc index 02a9c09d4c..820620712e 100644 --- a/docs/src/upgrade/release-3.2.x-incubating.asciidoc +++ b/docs/src/upgrade/release-3.2.x-incubating.asciidoc @@ -17,7 +17,7 @@ specific language governing permissions and limitations under the License. //// -[llms-summary="Upgrade notes for the TinkerPop 3.2.x release line: breaking changes, new features, and provider migration guidance across its releases."] +[llms-summary="Upgrade notes for the TinkerPop 3.2.x release line: breaking changes, new features, and provider migration guidance across its releases.",allow-oversize="true"] = TinkerPop 3.2.0 image::nine-inch-gremlins.png[width=225] diff --git a/docs/tinkeradoc-extension/src/main/java/org/apache/tinkerpop/tinkeradoc/MarkdownConverter.java b/docs/tinkeradoc-extension/src/main/java/org/apache/tinkerpop/tinkeradoc/MarkdownConverter.java index fa6de1ff9d..e7b617b503 100644 --- a/docs/tinkeradoc-extension/src/main/java/org/apache/tinkerpop/tinkeradoc/MarkdownConverter.java +++ b/docs/tinkeradoc-extension/src/main/java/org/apache/tinkerpop/tinkeradoc/MarkdownConverter.java @@ -55,18 +55,12 @@ public class MarkdownConverter extends StringConverter { static final String LLMS_SUMMARY_ATTR = "llms-summary"; /** - * Section attribute ({@code [llms-explode]}) marking a catalog section whose direct subsections - * should each become their own split page. Emitted as a hidden {@code <!-- llms-explode -->} - * marker for {@link MarkdownSplitter}; never rendered into the page body. + * Section attribute ({@code allow-oversize="true"}) marking a summarized page that may exceed the + * size budget without failing the build's size lint. Emitted as a hidden + * {@code <!-- llms-allow-oversize -->} marker for {@link MarkdownSplitter}; never rendered into + * the page body. Splitting itself is driven solely by {@code llms-summary}. */ - static final String LLMS_EXPLODE_ATTR = "llms-explode"; - - /** - * Section attribute ({@code [llms-keep]}) marking a section whose whole subtree must stay on one - * split page even if it exceeds the size budget. Emitted as a hidden {@code <!-- llms-keep -->} - * marker for {@link MarkdownSplitter}; never rendered into the page body. - */ - static final String LLMS_KEEP_ATTR = "llms-keep"; + static final String LLMS_ALLOW_OVERSIZE_ATTR = "allow-oversize"; public MarkdownConverter(final String backend, final Map<String, Object> opts) { super(backend, opts); @@ -214,38 +208,22 @@ public class MarkdownConverter extends StringConverter { appendAnchor(sb, section.getId()); sb.append(hashes).append(' ').append(section.getTitle()).append("\n\n"); appendLlmsSummary(sb, section.getAttribute(LLMS_SUMMARY_ATTR)); - appendExplodeMarker(sb, section.getAttribute(LLMS_EXPLODE_ATTR)); - appendKeepMarker(sb, section.getAttribute(LLMS_KEEP_ATTR)); + appendAllowOversizeMarker(sb, section.getAttribute(LLMS_ALLOW_OVERSIZE_ATTR)); sb.append(section.getContent()); return sb.toString(); } /** - * Emits a hidden {@code <!-- llms-keep -->} marker when a section carries the {@code [llms-keep]} - * attribute, telling {@link MarkdownSplitter} to keep this section's whole subtree on a single - * page even if it exceeds the size budget (used e.g. to keep each GraphSON version intact). - * Invisible in rendered output. - */ - private static void appendKeepMarker(final StringBuilder sb, final Object keep) { - if (keep == null) return; - final String v = keep.toString().trim(); - if (v.equalsIgnoreCase("false")) return; - sb.append("<!-- llms-keep -->\n\n"); - } - - /** - * Emits a hidden {@code <!-- llms-explode -->} marker when a section carries the - * {@code [llms-explode]} attribute. The marker tells {@link MarkdownSplitter} to give each direct - * subsection of this section its own page (rather than size-packing them), which is what turns a - * flat catalog such as the traversal step reference into one page per step. Invisible in rendered - * output, like the summary comment. + * Emits a hidden {@code <!-- llms-allow-oversize -->} marker when a section carries + * {@code allow-oversize="true"}. It tells the {@link MarkdownSplitter} size lint that this page + * (a summarized section whose subtree exceeds the byte budget) is an intentional exception and + * must not be reported as a violation. Invisible in rendered output. */ - private static void appendExplodeMarker(final StringBuilder sb, final Object explode) { - if (explode == null) return; - // Any non-empty/non-"false" value enables it; the bare [llms-explode] form yields "". - final String v = explode.toString().trim(); - if (v.equalsIgnoreCase("false")) return; - sb.append("<!-- llms-explode -->\n\n"); + private static void appendAllowOversizeMarker(final StringBuilder sb, final Object allow) { + if (allow == null) return; + final String v = allow.toString().trim(); + if (v.isEmpty() || v.equalsIgnoreCase("false")) return; + sb.append("<!-- llms-allow-oversize -->\n\n"); } /** Emits {@code <a id="..."></a>} on its own line when the node has a non-empty id. */ diff --git a/docs/tinkeradoc-extension/src/main/java/org/apache/tinkerpop/tinkeradoc/MarkdownSplitter.java b/docs/tinkeradoc-extension/src/main/java/org/apache/tinkerpop/tinkeradoc/MarkdownSplitter.java index 7a25d84631..fcc65ba381 100644 --- a/docs/tinkeradoc-extension/src/main/java/org/apache/tinkerpop/tinkeradoc/MarkdownSplitter.java +++ b/docs/tinkeradoc-extension/src/main/java/org/apache/tinkerpop/tinkeradoc/MarkdownSplitter.java @@ -49,25 +49,25 @@ import java.util.regex.Pattern; */ class MarkdownSplitter { - /** Default per-page budget in bytes (~50 KB), per the agent-docs spec's page-size check. */ + /** + * Per-page size budget in bytes (~50 KB), per the agent-docs spec's page-size check. Splitting is + * summary-driven, not size-driven; this budget is only used by the size lint to flag pages that + * exceed it and are not marked {@code allow-oversize}. + */ static final int DEFAULT_BUDGET = 50_000; - /** Safety margin (bytes) held back from the packing budget to absorb estimate/render drift. */ - private static final int PACK_SAFETY_MARGIN = 512; - private static final Pattern ANCHOR = Pattern.compile("^<a id=\"([^\"]+)\"></a>$"); private static final Pattern HEADING = Pattern.compile("^(#{1,6}) +(.*)$"); // Intra-document links: [label](#anchor). Capture label and anchor separately. private static final Pattern INTRA_LINK = Pattern.compile("\\]\\(#([^)]+)\\)"); - // Marker MarkdownConverter emits from [llms-explode]: split this section's children per-page. - private static final String EXPLODE_MARKER = "<!-- llms-explode -->"; - // Marker MarkdownConverter emits from [llms-keep]: keep this section's whole subtree on one page. - private static final String KEEP_MARKER = "<!-- llms-keep -->"; + // The sole page-break signal: the hidden marker MarkdownConverter emits from [llms-summary="..."]. + private static final Pattern SUMMARY_MARKER = Pattern.compile("^<!-- llms-summary: .* -->$"); + // Marker MarkdownConverter emits from allow-oversize="true": this page may exceed the budget. + private static final String ALLOW_OVERSIZE_MARKER = "<!-- llms-allow-oversize -->"; private static final Logger LOG = Logger.getLogger(MarkdownSplitter.class.getName()); private final int budget; - private final int packBudget; MarkdownSplitter() { this(DEFAULT_BUDGET); @@ -75,12 +75,6 @@ class MarkdownSplitter { MarkdownSplitter(final int budget) { this.budget = budget; - // Reserve room in the packing budget for (a) the llms.txt pointer prepended to every page - // after packing, and (b) a small safety margin. The per-node byte estimate can drift a few - // bytes from the concatenated render (newline/preamble joins), and pages pack right up to - // the limit; the margin keeps the final UTF-8 page comfortably under the hard cap. - final int pointer = LLMS_POINTER.getBytes(java.nio.charset.StandardCharsets.UTF_8).length; - this.packBudget = Math.max(1, budget - pointer - PACK_SAFETY_MARGIN); } /** @@ -93,6 +87,18 @@ class MarkdownSplitter { * @return the file names written into the book's directory */ List<String> splitFile(final Path bookFile) throws IOException { + return splitFile(bookFile, new ArrayList<>()); + } + + /** + * Splits {@code bookFile} in place and appends a human-readable message to {@code violations} for + * every written page that exceeds the byte budget and is not flagged {@code allow-oversize}. + * Splitting is summary-driven; the budget is enforced only as a lint (the caller decides whether + * violations fail the build). + * + * @return the file names written into the book's directory + */ + List<String> splitFile(final Path bookFile, final List<String> violations) throws IOException { final String markdown = new String(Files.readAllBytes(bookFile), StandardCharsets.UTF_8); final Path dir = bookFile.getParent(); final String indexName = bookFile.getFileName().toString(); @@ -104,9 +110,10 @@ class MarkdownSplitter { final byte[] bytes = page.getContent().getBytes(StandardCharsets.UTF_8); Files.write(out, bytes); written.add(page.getFileName()); - if (bytes.length > budget) { - LOG.warning("Markdown page " + out + " is " + bytes.length - + " bytes, over the " + budget + "-byte budget (indivisible single-heading section)"); + if (bytes.length > budget && !page.isOversizeAllowed()) { + violations.add(out + " is " + bytes.length + " bytes, over the " + budget + + "-byte budget; add an llms-summary to a subsection to split it, " + + "or mark the section allow-oversize=\"true\" if it must stay whole"); } } LOG.info("Split " + bookFile + " into " + written.size() + " pages: " + written); @@ -114,42 +121,59 @@ class MarkdownSplitter { } /** - * CLI entry point: {@code MarkdownSplitter [--budget N] <book.md> [<book.md> ...]}. Each named - * rendered book file is split in place into agent-sized pages in its own directory. + * CLI entry point: {@code MarkdownSplitter [--budget N] [--strict] <book.md> [<book.md> ...]}. + * Each named rendered book file is split in place (summary-driven) into pages in its own + * directory. Pages over the byte budget that are not flagged {@code allow-oversize} are reported; + * with {@code --strict} the process exits non-zero when any such violation exists, so the docs + * build can gate on it. */ public static void main(final String[] args) throws IOException { int budget = DEFAULT_BUDGET; + boolean strict = false; final List<String> files = new ArrayList<>(); for (int i = 0; i < args.length; i++) { if ("--budget".equals(args[i]) && i + 1 < args.length) { budget = Integer.parseInt(args[++i]); + } else if ("--strict".equals(args[i])) { + strict = true; } else { files.add(args[i]); } } if (files.isEmpty()) { - System.err.println("usage: MarkdownSplitter [--budget N] <book.md> [<book.md> ...]"); + System.err.println("usage: MarkdownSplitter [--budget N] [--strict] <book.md> [<book.md> ...]"); System.exit(2); } final MarkdownSplitter splitter = new MarkdownSplitter(budget); + final List<String> violations = new ArrayList<>(); for (final String f : files) { final Path p = Path.of(f); if (Files.isRegularFile(p)) { - splitter.splitFile(p); + splitter.splitFile(p, violations); } else { System.err.println("skip (not a file): " + f); } } + if (!violations.isEmpty()) { + System.err.println("\nMarkdown page-size budget violations (" + violations.size() + "):"); + for (final String v : violations) System.err.println(" - " + v); + if (strict) { + System.err.println("\nFailing due to --strict. Curate the offending sections and re-run."); + System.exit(1); + } + } } - /** A rendered page: its file name (without directory), and its Markdown body. */ + /** A rendered page: its file name (without directory), its Markdown body, and its oversize flag. */ static final class Page { final String fileName; final String content; + final boolean oversizeAllowed; - Page(final String fileName, final String content) { + Page(final String fileName, final String content, final boolean oversizeAllowed) { this.fileName = fileName; this.content = content; + this.oversizeAllowed = oversizeAllowed; } String getFileName() { @@ -159,6 +183,10 @@ class MarkdownSplitter { String getContent() { return content; } + + boolean isOversizeAllowed() { + return oversizeAllowed; + } } /** A parsed heading unit: its level, anchor id (may be null), and the raw block of lines. */ @@ -167,7 +195,6 @@ class MarkdownSplitter { final String anchor; final List<String> lines = new ArrayList<>(); final List<Node> children = new ArrayList<>(); - int byteSize; // size of this node's own lines plus descendants, computed lazily Node(final int level, final String anchor) { this.level = level; @@ -187,32 +214,84 @@ class MarkdownSplitter { */ List<Page> split(final String markdown, final String indexFileName) { final Node root = parse(markdown); - computeSizes(root); - // Decide which nodes start their own page. The root's own lines (preamble) always live on - // the index page. Each top-level child is placed on the index page if it fits; otherwise it - // (or its children, recursively) become separate pages. + // Summary-driven splitting: a section becomes its own page if and only if it carries an + // llms-summary (surfaced as the <!-- llms-summary: ... --> marker). The document root is + // always a page (the index). A page holds its owning node's content plus every descendant + // up to — but not including — the next summarized descendant, which breaks off to its own + // page. There is no size-based splitting; page size is an author concern surfaced by the + // build's size lint, with intentional exceptions flagged via allow-oversize. final Map<String, String> anchorToFile = new LinkedHashMap<>(); final List<PagePlan> plans = new ArrayList<>(); - final PagePlan index = new PagePlan(indexFileName); + final PagePlan index = new PagePlan(indexFileName, root); plans.add(index); + recordAnchorsUntilBreak(root, index.fileName, anchorToFile); + planBreaks(root, plans, anchorToFile); - // Record every anchor's home page as we assign, so links can be rewritten afterward. - assignRootAnchors(root, index, anchorToFile); - final PageCursor cursor = new PageCursor(index); - cursor.used = preambleSize(root); - planChildren(root, cursor, plans, anchorToFile); - - // Render each planned page, rewrite cross-page links, and prepend the llms.txt pointer. final List<Page> pages = new ArrayList<>(); for (final PagePlan plan : plans) { - final String body = plan.render(); - final String rewritten = rewriteLinks(body, plan.fileName, anchorToFile); - pages.add(new Page(plan.fileName, LLMS_POINTER + rewritten)); + final StringBuilder body = new StringBuilder(); + renderPage(plan.owner, body, plan == index); + final String rewritten = rewriteLinks(body.toString(), plan.fileName, anchorToFile); + pages.add(new Page(plan.fileName, LLMS_POINTER + rewritten, isAllowOversize(plan.owner))); } return pages; } + /** + * Walks the tree creating a page for every summarized descendant of {@code node} (the node's own + * page having already been created). Recurses through the whole tree so nested summarized + * sections each get a page. + */ + private void planBreaks(final Node node, final List<PagePlan> plans, + final Map<String, String> anchorToFile) { + for (final Node child : node.children) { + if (hasSummary(child)) { + final PagePlan page = new PagePlan(fileNameFor(child, plans), child); + plans.add(page); + recordAnchorsUntilBreak(child, page.fileName, anchorToFile); + } + planBreaks(child, plans, anchorToFile); + } + } + + /** + * Records that {@code node} and all descendants that render on the same page (i.e. everything + * down to the next summarized section) resolve to {@code fileName}, so links can be rewritten. + */ + private void recordAnchorsUntilBreak(final Node node, final String fileName, + final Map<String, String> anchorToFile) { + if (node.anchor != null) anchorToFile.put(node.anchor, fileName); + for (final String line : node.lines) { + final Matcher am = ANCHOR.matcher(line); + if (am.matches()) anchorToFile.put(am.group(1), fileName); + } + for (final Node child : node.children) { + if (hasSummary(child)) continue; // child owns its own page; recorded when its page is made + recordAnchorsUntilBreak(child, fileName, anchorToFile); + } + } + + /** + * Renders one page: {@code owner}'s own lines, then each descendant up to the next summarized + * section. Summarized descendants are omitted here (they render on their own page). + */ + private void renderPage(final Node owner, final StringBuilder sb, final boolean isIndex) { + for (final String l : owner.lines) sb.append(l).append('\n'); + for (final Node child : owner.children) { + if (hasSummary(child)) continue; + renderSubtree(child, sb); + } + } + + private void renderSubtree(final Node node, final StringBuilder sb) { + for (final String l : node.lines) sb.append(l).append('\n'); + for (final Node child : node.children) { + if (hasSummary(child)) continue; + renderSubtree(child, sb); + } + } + /** * The agent-facing directive prepended to every page (agentdocsspec.com {@code * llms-txt-directive-md} check): a top-of-page blockquote pointing at the site-root index. @@ -276,201 +355,35 @@ class MarkdownSplitter { return root; } - private int computeSizes(final Node node) { - int size = ownSize(node); - for (final Node c : node.children) size += computeSizes(c); - node.byteSize = size; - return size; - } - - /** - * The size of a node's own lines (heading + leading content) in UTF-8 bytes, excluding children. - * Uses UTF-8 byte length (not {@code String.length()}, which counts UTF-16 chars) because pages - * are written and size-checked as UTF-8; docs contain multi-byte characters (™, →, é, …), so a - * char count would under-estimate and let a page slip over the byte budget. - */ - private static int ownSize(final Node node) { - int size = 0; - for (final String l : node.lines) { - size += l.getBytes(java.nio.charset.StandardCharsets.UTF_8).length + 1; - } - return size; - } - - /** The byte size of the document preamble (root's own lines) that always leads the index page. */ - private static int preambleSize(final Node root) { - return ownSize(root); - } - - // ---- page planning ----------------------------------------------------- + // ---- page planning (summary-driven) ------------------------------------ + /** A planned page: its file name and the node whose subtree (up to the next break) it renders. */ private static final class PagePlan { final String fileName; - final List<Node> nodes = new ArrayList<>(); - final List<String> preambleLines = new ArrayList<>(); + final Node owner; - PagePlan(final String fileName) { + PagePlan(final String fileName, final Node owner) { this.fileName = fileName; - } - - String render() { - final StringBuilder sb = new StringBuilder(); - for (final String l : preambleLines) sb.append(l).append('\n'); - for (final Node n : nodes) renderNode(n, sb); - return sb.toString(); - } - - private void renderNode(final Node n, final StringBuilder sb) { - for (final String l : n.lines) sb.append(l).append('\n'); - for (final Node c : n.children) renderNode(c, sb); - } - } - - /** The root's own (pre-heading) lines and any anchors on them belong to the index page. */ - private void assignRootAnchors(final Node root, final PagePlan index, - final Map<String, String> anchorToFile) { - index.preambleLines.addAll(root.lines); - // Anchors that appear in the preamble lines resolve to the index page. - for (final String line : root.lines) { - final Matcher am = ANCHOR.matcher(line); - if (am.matches()) anchorToFile.put(am.group(1), index.fileName); + this.owner = owner; } } - /** Mutable pointer to the page currently being filled, plus its running byte size. */ - private static final class PageCursor { - PagePlan page; - int used; - - PageCursor(final PagePlan page) { - this.page = page; - } - } - - /** - * Greedily packs each child of {@code parent} onto the current page, keeping pages as full as - * possible under the budget while never cutting mid-section: - * <ul> - * <li>If the child's whole subtree fits in the current page's remaining room, place it there.</li> - * <li>Else, if the whole subtree fits an empty page, start a fresh page for it.</li> - * <li>Else (the subtree alone exceeds the budget), open a page led by the child's own heading - * and descend a level, recursively planning its children.</li> - * </ul> - * The descend case is what turns an over-budget chapter into per-section pages. - */ - private void planChildren(final Node parent, final PageCursor cursor, - final List<PagePlan> plans, final Map<String, String> anchorToFile) { - for (final Node child : parent.children) { - if (isExplode(child)) { - // Catalog section (e.g. the traversal step reference): give the section's own - // heading/preamble its own page, then put EACH direct subsection on its own page so - // every entry (each step) is individually named and addressable in llms.txt. - final PagePlan page = newPage(child, plans); - page.nodes.add(headOnly(child, page, anchorToFile)); - for (final Node grandchild : child.children) { - final PagePlan gcPage = newPage(grandchild, plans); - placeWhole(grandchild, gcPage, anchorToFile); - } - // Resume packing subsequent siblings on a fresh cursor (the catalog pages are done). - cursor.page = page; - cursor.used = packBudget; // force the next sibling onto its own page/flow - continue; - } - if (isKeep(child)) { - // Keep-whole section (e.g. a GraphSON version): emit the entire subtree as one page, - // never descending, even if it exceeds the budget. Use it as-is when it fits the - // current page's remaining room; otherwise give it its own page. - if (cursor.used + child.byteSize <= packBudget) { - placeWhole(child, cursor.page, anchorToFile); - cursor.used += child.byteSize; - } else { - final PagePlan page = newPage(child, plans); - placeWhole(child, page, anchorToFile); - cursor.page = page; - cursor.used = Math.max(child.byteSize, packBudget); // next sibling starts fresh - } - continue; - } - if (cursor.used + child.byteSize <= packBudget) { - placeWhole(child, cursor.page, anchorToFile); - cursor.used += child.byteSize; - } else if (child.byteSize <= packBudget) { - // Fits a page of its own: start a fresh page and move the cursor there. - final PagePlan page = newPage(child, plans); - placeWhole(child, page, anchorToFile); - cursor.page = page; - cursor.used = child.byteSize; - } else { - // Too big for any single page: lead a new page with the child's own heading, then - // descend into its children on that same page (they will overflow to more pages). - final PagePlan page = newPage(child, plans); - page.nodes.add(headOnly(child, page, anchorToFile)); - final PageCursor childCursor = new PageCursor(page); - childCursor.used = ownSize(child); - planChildren(child, childCursor, plans, anchorToFile); - // Continue the outer loop on the last page the descent produced, so a small sibling - // after a big chapter can still share that page's leftover room. - cursor.page = childCursor.page; - cursor.used = childCursor.used; - } - } - } - - private PagePlan newPage(final Node node, final List<PagePlan> plans) { - final PagePlan page = new PagePlan(fileNameFor(node, plans)); - plans.add(page); - return page; - } - - /** - * Whether a node is a catalog section marked for per-child explosion (its own lines contain the - * {@code <!-- llms-explode -->} marker emitted from the {@code [llms-explode]} attribute) and it - * actually has children to explode. - */ - private static boolean isExplode(final Node node) { - if (node.children.isEmpty()) return false; + /** Whether a node carries an llms-summary (the sole page-break signal). */ + private static boolean hasSummary(final Node node) { for (final String line : node.lines) { - if (line.trim().equals(EXPLODE_MARKER)) return true; + if (SUMMARY_MARKER.matcher(line.trim()).matches()) return true; } return false; } - /** - * Whether a node is marked keep-whole (its own lines contain the {@code <!-- llms-keep -->} - * marker emitted from the {@code [llms-keep]} attribute). Such a node's entire subtree is emitted - * as a single page and is never descended into, even when it exceeds the byte budget. - */ - private static boolean isKeep(final Node node) { + /** Whether a node is flagged to permit exceeding the size budget without a lint failure. */ + static boolean isAllowOversize(final Node node) { for (final String line : node.lines) { - if (line.trim().equals(KEEP_MARKER)) return true; + if (line.trim().equals(ALLOW_OVERSIZE_MARKER)) return true; } return false; } - /** Places a node (and its whole subtree) onto a page, recording all its anchors' home. */ - private void placeWhole(final Node node, final PagePlan page, final Map<String, String> anchorToFile) { - page.nodes.add(node); - recordAnchors(node, page.fileName, anchorToFile); - } - - /** - * Returns a shallow copy of {@code node} holding only its own lines (heading + leading content), - * with its children dropped, so the node's heading leads its own page while its subsections are - * planned separately. Records the node's own anchor on the given page. - */ - private Node headOnly(final Node node, final PagePlan page, final Map<String, String> anchorToFile) { - final Node head = new Node(node.level, node.anchor); - head.lines.addAll(node.lines); - if (node.anchor != null) anchorToFile.put(node.anchor, page.fileName); - return head; - } - - private void recordAnchors(final Node node, final String fileName, - final Map<String, String> anchorToFile) { - if (node.anchor != null) anchorToFile.put(node.anchor, fileName); - for (final Node c : node.children) recordAnchors(c, fileName, anchorToFile); - } - private String fileNameFor(final Node node, final List<PagePlan> plans) { final String base = node.anchor != null && !node.anchor.isEmpty() ? sanitize(node.anchor) : "section"; diff --git a/docs/tinkeradoc-extension/src/test/java/org/apache/tinkerpop/tinkeradoc/MarkdownConverterProbeTest.java b/docs/tinkeradoc-extension/src/test/java/org/apache/tinkerpop/tinkeradoc/MarkdownConverterProbeTest.java index 9a4780303d..52cbe0c6e4 100644 --- a/docs/tinkeradoc-extension/src/test/java/org/apache/tinkerpop/tinkeradoc/MarkdownConverterProbeTest.java +++ b/docs/tinkeradoc-extension/src/test/java/org/apache/tinkerpop/tinkeradoc/MarkdownConverterProbeTest.java @@ -108,25 +108,21 @@ public class MarkdownConverterProbeTest { } @Test - public void llmsExplodeAttributeBecomesHiddenMarkerNotBodyText() { - // [llms-explode] on a catalog section emits a hidden <!-- llms-explode --> marker for the - // splitter, and must not appear as visible body text. - final String md = toMarkdown("= T\n\n[llms-explode=\"\"]\n== Steps\n\nCatalog intro.\n\n=== A Step\n\nBody.\n"); - assertThat(md, containsString("<!-- llms-explode -->")); - assertThat(md, containsString("Catalog intro.")); - } - - @Test - public void noLlmsExplodeMarkerWhenAttributeAbsent() { - final String md = toMarkdown("= T\n\n== Steps\n\nCatalog intro.\n"); - assertThat(md, not(containsString("llms-explode"))); + public void allowOversizeAttributeBecomesHiddenMarkerNotBodyText() { + // allow-oversize="true" on a section emits a hidden <!-- llms-allow-oversize --> marker for + // the splitter's size lint, and must not appear as visible body text. + final String md = toMarkdown("= T\n\n[llms-summary=\"GraphSON 2.0.\",allow-oversize=\"true\"]\n" + + "== Version 2.0\n\nVersion body.\n"); + assertThat(md, containsString("<!-- llms-allow-oversize -->")); + assertThat(md, containsString("Version body.")); + assertThat(md, not(containsString("allow-oversize="))); } @Test - public void llmsKeepAttributeBecomesHiddenMarkerNotBodyText() { - final String md = toMarkdown("= T\n\n[llms-keep=\"\"]\n== Version 2.0\n\nVersion body.\n"); - assertThat(md, containsString("<!-- llms-keep -->")); - assertThat(md, containsString("Version body.")); + public void noAllowOversizeMarkerWhenAttributeAbsentOrFalse() { + assertThat(toMarkdown("= T\n\n== Version 2.0\n\nBody.\n"), not(containsString("allow-oversize"))); + assertThat(toMarkdown("= T\n\n[allow-oversize=\"false\"]\n== V\n\nBody.\n"), + not(containsString("llms-allow-oversize"))); } @Test diff --git a/docs/tinkeradoc-extension/src/test/java/org/apache/tinkerpop/tinkeradoc/MarkdownSplitterTest.java b/docs/tinkeradoc-extension/src/test/java/org/apache/tinkerpop/tinkeradoc/MarkdownSplitterTest.java index 28e0d60771..67b2810523 100644 --- a/docs/tinkeradoc-extension/src/test/java/org/apache/tinkerpop/tinkeradoc/MarkdownSplitterTest.java +++ b/docs/tinkeradoc-extension/src/test/java/org/apache/tinkerpop/tinkeradoc/MarkdownSplitterTest.java @@ -30,17 +30,25 @@ import static org.hamcrest.CoreMatchers.not; import static org.hamcrest.MatcherAssert.assertThat; /** - * Tests for {@link MarkdownSplitter}: the size-driven, heading-aligned page splitter and its - * cross-page link rewriting. + * Tests for {@link MarkdownSplitter}, which splits a rendered book into pages driven solely by + * {@code llms-summary} markers: a section becomes its own page iff it carries a summary. Unsummarized + * sections attach to their nearest summarized ancestor's page. There is no size-based splitting; the + * byte budget is only a lint, with intentional exceptions flagged {@code allow-oversize}. */ public class MarkdownSplitterTest { + /** A heading with an anchor, no summary (does NOT start a page). */ private static String heading(final String id, final int level, final String title) { final StringBuilder h = new StringBuilder(); for (int i = 0; i < level; i++) h.append('#'); return "<a id=\"" + id + "\"></a>\n" + h + " " + title + "\n"; } + /** A heading with an anchor and an llms-summary marker (starts its own page). */ + private static String summarized(final String id, final int level, final String title, final String summary) { + return heading(id, level, title) + "<!-- llms-summary: " + summary + " -->\n"; + } + private static String filler(final int bytes) { final StringBuilder sb = new StringBuilder(); while (sb.length() < bytes) sb.append("lorem ipsum dolor sit amet consectetur\n"); @@ -51,203 +59,125 @@ public class MarkdownSplitterTest { return pages.stream().map(MarkdownSplitter.Page::getFileName).collect(Collectors.toList()); } - @Test - public void smallBookStaysSinglePage() { - final String md = heading("_io_reference", 1, "IO Reference") + "\nsome intro\n\n" - + heading("graphml", 1, "GraphML") + "\nshort content\n"; - final List<MarkdownSplitter.Page> pages = new MarkdownSplitter(50_000).split(md, "index.md"); - assertThat(pages.size(), is(1)); - assertThat(pages.get(0).getFileName(), is("index.md")); - assertThat(pages.get(0).getContent(), containsString("# GraphML")); + private static MarkdownSplitter.Page page(final List<MarkdownSplitter.Page> pages, final String name) { + return pages.stream().filter(p -> p.getFileName().equals(name)).findFirst() + .orElseThrow(() -> new AssertionError("no page named " + name + " in " + fileNames(pages))); } + // A "book" in rendered Markdown is a flat sequence of level-1 (#) sections: the doctitle first, + // then chapters as siblings (AsciiDoc renders a book's doctitle and its level-0 chapters all as + // <h1>). The document root (index.md) owns the preamble and the doctitle section's content up to + // the first summarized sibling. + @Test - public void oversizedSectionBecomesOwnPage() { - final String md = heading("_io_reference", 1, "IO Reference") + "\nintro\n\n" - + heading("graphml", 1, "GraphML") + "\nshort\n\n" - + heading("graphson", 1, "GraphSON") + "\n" + filler(60_000); - final List<MarkdownSplitter.Page> pages = new MarkdownSplitter(50_000).split(md, "index.md"); - final List<String> names = fileNames(pages); - // The big GraphSON chapter must be split off into graphson.md; index keeps the small parts. - assertThat(names, hasItem("index.md")); - assertThat(names, hasItem("graphson.md")); - final MarkdownSplitter.Page index = pages.stream() - .filter(p -> p.getFileName().equals("index.md")).findFirst().orElseThrow(AssertionError::new); - assertThat(index.getContent(), containsString("# GraphML")); - assertThat(index.getContent(), not(containsString(filler(60_000).substring(0, 200)))); + public void bookWithNoSummariesIsASinglePage() { + final String md = "preamble\n\n" + heading("io", 1, "IO Reference") + "\nintro\n\n" + + heading("a", 1, "A") + "\ncontent a\n\n" + + heading("b", 1, "B") + "\ncontent b\n"; + final List<MarkdownSplitter.Page> pages = new MarkdownSplitter().split(md, "index.md"); + assertThat(pages.size(), is(1)); + assertThat(pages.get(0).getFileName(), is("index.md")); + assertThat(pages.get(0).getContent(), containsString("# A")); + assertThat(pages.get(0).getContent(), containsString("content b")); } @Test - public void everyDivisiblePageUnderBudget() { - // A, B, C each have small children? No — here each is a single leaf heading. B alone is - // 60KB of direct content, which is indivisible: it must occupy one page even though that - // page exceeds the budget (there is no finer heading to split on). Every OTHER page must - // be under budget. - final String md = heading("book", 1, "Book") + "\nintro\n\n" - + heading("a", 1, "A") + "\n" + filler(40_000) - + heading("b", 1, "B") + "\n" + filler(60_000) - + heading("c", 1, "C") + "\n" + filler(20_000); - final int budget = 50_000; - final List<MarkdownSplitter.Page> pages = new MarkdownSplitter(budget).split(md, "index.md"); - for (final MarkdownSplitter.Page p : pages) { - final boolean indivisible = p.getFileName().equals("b.md"); - if (!indivisible) { - assertThat(p.getFileName() + " over budget: " + p.getContent().length(), - p.getContent().length() <= budget, is(true)); - } - } - // The indivisible 60KB leaf is isolated on its own page (so it doesn't bloat neighbors). + public void eachSummarizedSectionBecomesItsOwnPage() { + final String md = "preamble\n\n" + heading("io", 1, "IO Reference") + "\nintro\n\n" + + summarized("a", 1, "A", "Section A.") + "\ncontent a\n\n" + + summarized("b", 1, "B", "Section B.") + "\ncontent b\n"; + final List<MarkdownSplitter.Page> pages = new MarkdownSplitter().split(md, "index.md"); + assertThat(fileNames(pages), hasItem("index.md")); + assertThat(fileNames(pages), hasItem("a.md")); assertThat(fileNames(pages), hasItem("b.md")); + assertThat(page(pages, "a.md").getContent(), containsString("content a")); + assertThat(page(pages, "a.md").getContent(), not(containsString("content b"))); + // The index holds the preamble + the (unsummarized) doctitle section, not the summarized ones. + assertThat(page(pages, "index.md").getContent(), containsString("intro")); + assertThat(page(pages, "index.md").getContent(), not(containsString("content a"))); } @Test - public void renderedPageIncludingPointerStaysUnderBudget() { - // Every divisible page's FINAL content (which includes the prepended llms.txt pointer) must - // stay within the budget: the splitter reserves room for the pointer when packing. - final int budget = 50_000; - final StringBuilder md = new StringBuilder(heading("book", 1, "Book") + "\nintro\n\n"); - // Many medium sections that would pack right up against the budget. - for (int i = 0; i < 8; i++) { - md.append(heading("sec-" + i, 1, "Section " + i)).append('\n').append(filler(12_000)); - } - final List<MarkdownSplitter.Page> pages = new MarkdownSplitter(budget).split(md.toString(), "index.md"); - for (final MarkdownSplitter.Page p : pages) { - assertThat(p.getFileName() + " (incl pointer) = " + p.getContent().length(), - p.getContent().length() <= budget, is(true)); - assertThat(p.getContent(), containsString("[llms.txt](/llms.txt)")); - } + public void unsummarizedChildAttachesToSummarizedAncestorPage() { + // Chapter is summarized (own page); its child A1 is not (stays on chapter page); its child A2 + // is summarized (breaks off). + final String md = "preamble\n\n" + heading("io", 1, "IO Reference") + "\nintro\n\n" + + summarized("chapter", 1, "Chapter", "A chapter.") + "\nlead\n\n" + + heading("a1", 2, "A1") + "\ncontent a1\n\n" + + summarized("a2", 2, "A2", "Subsection A2.") + "\ncontent a2\n"; + final List<MarkdownSplitter.Page> pages = new MarkdownSplitter().split(md, "index.md"); + assertThat(fileNames(pages), hasItem("chapter.md")); + assertThat(fileNames(pages), hasItem("a2.md")); + assertThat(fileNames(pages), not(hasItem("a1.md"))); + assertThat(page(pages, "chapter.md").getContent(), containsString("## A1")); + assertThat(page(pages, "chapter.md").getContent(), containsString("content a1")); + assertThat(page(pages, "chapter.md").getContent(), not(containsString("content a2"))); + assertThat(page(pages, "a2.md").getContent(), containsString("content a2")); } @Test - public void indivisibleOversizedLeafIsIsolatedOnOwnPage() { - final String md = heading("book", 1, "Book") + "\nintro\n\n" - + heading("small", 1, "Small") + "\nshort\n\n" - + heading("huge", 1, "Huge") + "\n" + filler(70_000); - final List<MarkdownSplitter.Page> pages = new MarkdownSplitter(50_000).split(md, "index.md"); - final MarkdownSplitter.Page huge = pages.stream() - .filter(p -> p.getFileName().equals("huge.md")).findFirst().orElseThrow(AssertionError::new); - // The huge leaf page contains only the Huge section, not the small one. - assertThat(huge.getContent(), containsString("# Huge")); - assertThat(huge.getContent(), not(containsString("# Small"))); + public void catalogWithPerChildSummariesYieldsOnePagePerChild() { + // A "catalog" is simply a summarized parent whose children are each summarized: no special + // marker needed — summaries alone drive the per-child split. + final String md = "preamble\n\n" + heading("io", 1, "IO Reference") + "\nintro\n\n" + + summarized("steps", 1, "Steps", "The catalog.") + "\ncatalog intro\n\n" + + summarized("fold-step", 2, "Fold Step", "fold() aggregates.") + "\nfold body\n\n" + + summarized("group-step", 2, "Group Step", "group() organizes.") + "\ngroup body\n"; + final List<MarkdownSplitter.Page> pages = new MarkdownSplitter().split(md, "index.md"); + assertThat(fileNames(pages), hasItem("steps.md")); + assertThat(fileNames(pages), hasItem("fold-step.md")); + assertThat(fileNames(pages), hasItem("group-step.md")); + assertThat(page(pages, "fold-step.md").getContent(), containsString("fold body")); + assertThat(page(pages, "fold-step.md").getContent(), not(containsString("group body"))); + assertThat(page(pages, "steps.md").getContent(), containsString("catalog intro")); } @Test - public void descendsToChildrenWhenChapterTooBig() { - // A chapter (90KB total) that exceeds the budget descends to its h2 children. Greedy packing - // keeps pages full: the chapter page leads with the chapter heading + as many whole child - // sections as fit; the remainder overflow to further pages. So we get chapter.md (heading + - // Section One) and a second page for Section Two — never a mid-section cut. - final String md = heading("book", 1, "Book") + "\nintro\n\n" - + heading("chapter", 1, "Chapter") + "\nlead-in\n\n" - + heading("sec-one", 2, "Section One") + "\n" + filler(45_000) - + heading("sec-two", 2, "Section Two") + "\n" + filler(45_000); - final List<MarkdownSplitter.Page> pages = new MarkdownSplitter(50_000).split(md, "index.md"); - final List<String> names = fileNames(pages); - assertThat(names, hasItem("chapter.md")); - // Section Two overflows to its own page (named from its anchor); each h2 stays whole. - assertThat(names, hasItem("sec-two.md")); - final MarkdownSplitter.Page chapter = pages.stream() - .filter(p -> p.getFileName().equals("chapter.md")).findFirst().orElseThrow(AssertionError::new); - assertThat(chapter.getContent(), containsString("# Chapter")); - assertThat(chapter.getContent(), containsString("## Section One")); - assertThat(chapter.getContent(), not(containsString("## Section Two"))); + public void keepWholeIsJustAbsenceOfChildSummaries() { + // A summarized section whose (large) children are NOT summarized stays one whole page. + final String md = "preamble\n\n" + heading("io", 1, "IO Reference") + "\nintro\n\n" + + summarized("version-2", 1, "Version 2.0", "GraphSON 2.0.") + "\nversion intro\n\n" + + heading("edge", 2, "Edge") + "\n" + filler(30_000) + + heading("vertex", 2, "Vertex") + "\n" + filler(30_000); + final List<MarkdownSplitter.Page> pages = new MarkdownSplitter().split(md, "index.md"); + assertThat(fileNames(pages), hasItem("version-2.md")); + assertThat(fileNames(pages), not(hasItem("edge.md"))); + final MarkdownSplitter.Page v2 = page(pages, "version-2.md"); + assertThat(v2.getContent(), containsString("## Edge")); + assertThat(v2.getContent(), containsString("## Vertex")); } @Test public void rewritesCrossPageLinksAndKeepsSamePageBare() { - final String md = heading("book", 1, "Book") + "\nintro\n\n" - + heading("graphml", 1, "GraphML") + "\nSee [GraphSON](#graphson) and [self](#graphml).\n\n" - + heading("graphson", 1, "GraphSON") + "\n" + filler(60_000) - + "Back to [GraphML](#graphml).\n"; - final List<MarkdownSplitter.Page> pages = new MarkdownSplitter(50_000).split(md, "index.md"); - - final MarkdownSplitter.Page index = pages.stream() - .filter(p -> p.getFileName().equals("index.md")).findFirst().orElseThrow(AssertionError::new); - final MarkdownSplitter.Page graphson = pages.stream() - .filter(p -> p.getFileName().equals("graphson.md")).findFirst().orElseThrow(AssertionError::new); - - // GraphML lives on index; its link to GraphSON (another page) is rewritten; self-link stays bare. - assertThat(index.getContent(), containsString("[GraphSON](graphson.md#graphson)")); - assertThat(index.getContent(), containsString("[self](#graphml)")); - // GraphSON page links back to GraphML on the index page. - assertThat(graphson.getContent(), containsString("[GraphML](index.md#graphml)")); - } - - @Test - public void explodeMarkerGivesEachChildItsOwnPage() { - // A catalog section marked with <!-- llms-explode --> must split each direct child onto its - // own page (named from the child anchor), regardless of size — even though all children are - // tiny and would otherwise pack onto one page. - final String md = heading("book", 1, "Book") + "\nintro\n\n" - + heading("graph-traversal-steps", 1, "Graph Traversal Steps") + "\n<!-- llms-explode -->\n\ncatalog intro\n\n" - + heading("fold-step", 2, "Fold Step") + "\nThe fold() step.\n\n" - + heading("group-step", 2, "Group Step") + "\nThe group() step.\n\n" - + heading("count-step", 2, "Count Step") + "\nThe count() step.\n"; - final List<MarkdownSplitter.Page> pages = new MarkdownSplitter(50_000).split(md, "index.md"); - final List<String> names = fileNames(pages); - // Each step is its own page, plus the catalog head page. - assertThat(names, hasItem("graph-traversal-steps.md")); - assertThat(names, hasItem("fold-step.md")); - assertThat(names, hasItem("group-step.md")); - assertThat(names, hasItem("count-step.md")); - // The fold page contains only fold, not the others. - final MarkdownSplitter.Page fold = pages.stream() - .filter(p -> p.getFileName().equals("fold-step.md")).findFirst().orElseThrow(AssertionError::new); - assertThat(fold.getContent(), containsString("The fold() step.")); - assertThat(fold.getContent(), not(containsString("The group() step."))); - // The catalog head page keeps its intro but not the steps. - final MarkdownSplitter.Page cat = pages.stream() - .filter(p -> p.getFileName().equals("graph-traversal-steps.md")).findFirst().orElseThrow(AssertionError::new); - assertThat(cat.getContent(), containsString("catalog intro")); - assertThat(cat.getContent(), not(containsString("The fold() step."))); + final String md = "preamble\n\n" + heading("io", 1, "IO Reference") + "\nintro\n\n" + + summarized("graphml", 1, "GraphML", "GraphML.") + + "\nSee [GraphSON](#graphson) and [self](#graphml).\n\n" + + summarized("graphson", 1, "GraphSON", "GraphSON.") + "\nBack to [GraphML](#graphml).\n"; + final List<MarkdownSplitter.Page> pages = new MarkdownSplitter().split(md, "index.md"); + assertThat(page(pages, "graphml.md").getContent(), containsString("[GraphSON](graphson.md#graphson)")); + assertThat(page(pages, "graphml.md").getContent(), containsString("[self](#graphml)")); + assertThat(page(pages, "graphson.md").getContent(), containsString("[GraphML](graphml.md#graphml)")); } @Test - public void keepMarkerHoldsWholeSubtreeOnOnePageEvenOverBudget() { - // A keep-whole section must stay a single page even when its subtree exceeds the budget, - // rather than being descended/fragmented into per-child pages. - final String md = heading("book", 1, "Book") + "\nintro\n\n" - + heading("version-2", 1, "Version 2.0") + "\n<!-- llms-keep -->\n\nintro\n\n" - + heading("edge", 2, "Edge") + "\n" + filler(30_000) - + heading("vertex", 2, "Vertex") + "\n" + filler(30_000); - final List<MarkdownSplitter.Page> pages = new MarkdownSplitter(50_000).split(md, "index.md"); - final List<String> names = fileNames(pages); - // The whole Version 2.0 subtree is one page; its children did NOT become separate pages. - assertThat(names, hasItem("version-2.md")); - assertThat(names, not(hasItem("edge.md"))); - assertThat(names, not(hasItem("vertex.md"))); - final MarkdownSplitter.Page v2 = pages.stream() - .filter(p -> p.getFileName().equals("version-2.md")).findFirst().orElseThrow(AssertionError::new); - // Contains both children on the one page, and is (intentionally) over the budget. - assertThat(v2.getContent(), containsString("## Edge")); - assertThat(v2.getContent(), containsString("## Vertex")); - assertThat(v2.getContent().length() > 50_000, is(true)); + public void unknownAnchorsLeftUnchanged() { + final String md = "preamble\n\n" + heading("io", 1, "IO Reference") + + "\nSee [external](#not-a-heading).\n"; + final List<MarkdownSplitter.Page> pages = new MarkdownSplitter().split(md, "index.md"); + assertThat(pages.get(0).getContent(), containsString("[external](#not-a-heading)")); } @Test public void hashCommentsInsideCodeFenceAreNotTreatedAsHeadings() { - // A code fence containing shell/properties comment lines that begin with '#' must NOT be - // parsed as section headings (which would split the page mid-code-block and produce generic - // "section.md" file names). The whole fenced block stays with its owning section. - final String md = heading("book", 1, "Book") + "\nintro\n\n" - + heading("config", 1, "Config") + "\nHere is a config file:\n\n" + final String md = "preamble\n\n" + heading("io", 1, "IO Reference") + "\nintro\n\n" + + summarized("config", 1, "Config", "Config file.") + "\nHere is a config file:\n\n" + "```properties\n# Spark Configuration\nspark.master=local[4]\n# another comment\n" + "spark.executor.memory=1g\n```\n\nAfter the block.\n"; - final List<MarkdownSplitter.Page> pages = new MarkdownSplitter(50_000).split(md, "index.md"); - final List<String> names = fileNames(pages); - // Only index.md — no page split off a code comment, no generic "section*.md". - assertThat(names.stream().noneMatch(n -> n.startsWith("section")), is(true)); - final String body = pages.get(0).getContent(); - // The fenced block and its comments remain intact and are not promoted to headings. + final List<MarkdownSplitter.Page> pages = new MarkdownSplitter().split(md, "index.md"); + assertThat(fileNames(pages).stream().noneMatch(n -> n.startsWith("section")), is(true)); + final String body = page(pages, "config.md").getContent(); assertThat(body, containsString("# Spark Configuration")); assertThat(body, containsString("```properties")); assertThat(body, containsString("After the block.")); } - - @Test - public void unknownAnchorsLeftUnchanged() { - final String md = heading("book", 1, "Book") + "\nSee [external](#not-a-heading).\n"; - final List<MarkdownSplitter.Page> pages = new MarkdownSplitter(50_000).split(md, "index.md"); - assertThat(pages.get(0).getContent(), containsString("[external](#not-a-heading)")); - } }
