https://github.com/rnk updated https://github.com/llvm/llvm-project/pull/228590
>From b86a79f9960a7c093ca91ae5d8ce0771b3f7f1b7 Mon Sep 17 00:00:00 2001 From: Reid Kleckner <[email protected]> Date: Wed, 30 Sep 2026 19:54:06 +0000 Subject: [PATCH] [clang][docs] Generate split attribute reference docs Implements stage 1 of https://github.com/llvm/llvm-project/issues/227907 by teaching the attribute docs generator to emit split-file-formatted Markdown and wiring the docs build to split it for Sphinx. --- clang/docs/CMakeLists.txt | 37 +++- clang/docs/conf.py | 7 +- clang/test/TableGen/attr-docs.td | 32 ++++ clang/utils/TableGen/ClangAttrEmitter.cpp | 209 +++++++++++----------- 4 files changed, 172 insertions(+), 113 deletions(-) create mode 100644 clang/test/TableGen/attr-docs.td diff --git a/clang/docs/CMakeLists.txt b/clang/docs/CMakeLists.txt index 503ee47c9764c..d7f4cfcd485e0 100644 --- a/clang/docs/CMakeLists.txt +++ b/clang/docs/CMakeLists.txt @@ -84,12 +84,17 @@ function (gen_sphinx_file_from_td output_file td_option source docs_targets) if (NOT EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/${source}") message(FATAL_ERROR "Cannot find source file: ${source} in ${CMAKE_CURRENT_SOURCE_DIR}") endif() + get_filename_component(output_dir "${output_file}" DIRECTORY) + if (NOT "${output_dir}" STREQUAL "") + file(MAKE_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/${output_dir}") + endif() + string(REPLACE "/" "-" output_target "${output_file}") get_filename_component(TABLEGEN_INCLUDE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/${source}" DIRECTORY) list(APPEND LLVM_TABLEGEN_FLAGS "-I${TABLEGEN_INCLUDE_DIR}") - clang_tablegen(${output_file} ${td_option} SOURCE ${source} TARGET "gen-${output_file}") - add_dependencies("gen-${output_file}" copy-clang-rst-docs) + clang_tablegen(${output_file} ${td_option} SOURCE ${source} TARGET "gen-${output_target}") + add_dependencies("gen-${output_target}" copy-clang-rst-docs) foreach(target ${docs_targets}) - add_dependencies(${target} gen-${output_file}) + add_dependencies(${target} gen-${output_target}) endforeach() endfunction() @@ -104,6 +109,7 @@ if (LLVM_ENABLE_SPHINX) # directory before we run sphinx. set(clang_generated_docs AttributeReference.md + _generated/AttributeReference.split.md DiagnosticsReference.md AMDGPUBuiltinReference.rst ClangCommandLineReference.md @@ -111,6 +117,8 @@ if (LLVM_ENABLE_SPHINX) add_sphinx_source_sync_target(copy-clang-rst-docs "${CMAKE_CURRENT_SOURCE_DIR}" "${CMAKE_CURRENT_BINARY_DIR}" PRESERVE_DOCS ${clang_generated_docs}) + set(clang_docs_sync_stamp + "${CMAKE_CURRENT_BINARY_DIR}/CMakeFiles/copy-clang-rst-docs.sphinx-source-sync/sync.stamp") add_custom_command(TARGET copy-clang-rst-docs POST_BUILD COMMAND "${CMAKE_COMMAND}" -E copy_if_different "${CMAKE_CURRENT_SOURCE_DIR}/../Maintainers.md" @@ -145,7 +153,28 @@ if (LLVM_ENABLE_SPHINX) endif() # Generated files - gen_sphinx_file_from_td(AttributeReference.md -gen-attr-docs ../include/clang/Basic/Attr.td "${docs_targets}") + set(attr_reference_split_input "${CMAKE_CURRENT_BINARY_DIR}/_generated/AttributeReference.split.md") + set(attr_reference_split_stamp "${CMAKE_CURRENT_BINARY_DIR}/_generated/AttributeReferenceSplit.stamp") + gen_sphinx_file_from_td(_generated/AttributeReference.split.md -gen-attr-docs ../include/clang/Basic/Attr.td "") + add_custom_command( + OUTPUT "${attr_reference_split_stamp}" + COMMAND "${CMAKE_COMMAND}" -E rm -rf + "${CMAKE_CURRENT_BINARY_DIR}/AttributeReference" + COMMAND $<TARGET_FILE:split-file> + "${attr_reference_split_input}" + "${CMAKE_CURRENT_BINARY_DIR}" + COMMAND "${CMAKE_COMMAND}" -E touch + "${attr_reference_split_stamp}" + DEPENDS "${attr_reference_split_input}" + "${clang_docs_sync_stamp}" + split-file + VERBATIM + COMMENT "Splitting generated AttributeReference Markdown") + add_custom_target(gen-AttributeReferenceSplit + DEPENDS "${attr_reference_split_stamp}") + foreach(target ${docs_targets}) + add_dependencies(${target} gen-AttributeReferenceSplit) + endforeach() gen_sphinx_file_from_td(DiagnosticsReference.md -gen-diag-docs ../include/clang/Basic/Diagnostic.td "${docs_targets}") gen_sphinx_file_from_td(AMDGPUBuiltinReference.rst -gen-builtin-docs ../include/clang/Basic/BuiltinsAMDGPU.td "${docs_targets}") gen_sphinx_file_from_td(ClangCommandLineReference.md -gen-opt-docs ../include/clang/Options/ClangOptionDocs.td "${docs_targets}") diff --git a/clang/docs/conf.py b/clang/docs/conf.py index e70a5c1aeaf81..6cf4134f1f2bc 100644 --- a/clang/docs/conf.py +++ b/clang/docs/conf.py @@ -54,7 +54,12 @@ # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. -exclude_patterns = ["_build", "CIR/_raw/*"] +exclude_patterns = [ + "_build", + "_generated/*", + "AttributeReference/*", + "CIR/_raw/*", +] # The reST default role (used for this markup: `text`) to use for all documents. # default_role = None diff --git a/clang/test/TableGen/attr-docs.td b/clang/test/TableGen/attr-docs.td new file mode 100644 index 0000000000000..c60e7333f5c1a --- /dev/null +++ b/clang/test/TableGen/attr-docs.td @@ -0,0 +1,32 @@ +// RUN: clang-tblgen -gen-attr-docs -I%p/../../include %s -o - 2>&1 | FileCheck %s + +include "clang/Basic/Attr.td" + +def TestSplitDocCategory : DocumentationCategory<"Test Attributes"> { + let Content = [{ +Test category introduction. +}]; +} + +def TestSplitDocs : Documentation { + let Category = TestSplitDocCategory; + let Content = [{ +Test split documentation content. +}]; +} + +def TestSplitAttr : InheritableAttr { + let Spellings = [Clang<"test_split_attr">]; + let Subjects = SubjectList<[Function]>; + let Documentation = [TestSplitDocs]; +} + +// CHECK: //--- AttributeReference.md +// CHECK: :::{include} AttributeReference/TestAttributes.md +// CHECK-NEXT: ::: +// CHECK: //--- AttributeReference/TestAttributes.md +// CHECK: ## Test Attributes +// CHECK: Test category introduction. +// CHECK: ### test_split_attr +// CHECK: `test_split_attr` +// CHECK: Test split documentation content. diff --git a/clang/utils/TableGen/ClangAttrEmitter.cpp b/clang/utils/TableGen/ClangAttrEmitter.cpp index 59aac4c507772..4b03c759c2758 100644 --- a/clang/utils/TableGen/ClangAttrEmitter.cpp +++ b/clang/utils/TableGen/ClangAttrEmitter.cpp @@ -5471,18 +5471,18 @@ class SpellingList { } }; -class DocumentationData { -public: +struct DocumentationData { const Record *Documentation; - const Record *Attribute; + const Record *Category; + std::vector<const Record *> Attributes; std::string Heading; SpellingList SupportedSpellings; - DocumentationData(const Record &Documentation, const Record &Attribute, - std::pair<std::string, SpellingList> HeadingAndSpellings) - : Documentation(&Documentation), Attribute(&Attribute), - Heading(std::move(HeadingAndSpellings.first)), - SupportedSpellings(std::move(HeadingAndSpellings.second)) {} + DocumentationData(const Record &Documentation, + ArrayRef<const Record *> Attributes) + : Documentation(&Documentation), + Category(Documentation.getValueAsDef("Category")), + Attributes(Attributes) {} }; static void WriteCategoryHeader(const Record *DocCategory, @@ -5498,6 +5498,16 @@ static void WriteCategoryHeader(const Record *DocCategory, OS << "\n\n"; } +static std::string getAttributeReferenceCategoryFileName(StringRef Name) { + std::string FileName; + for (char C : Name) { + if (llvm::isAlnum(C)) + FileName += C; + } + FileName += ".md"; + return FileName; +} + static std::pair<std::string, SpellingList> GetAttributeHeadingAndSpellings(const Record &Documentation, const Record &Attribute, @@ -5559,6 +5569,24 @@ GetAttributeHeadingAndSpellings(const Record &Documentation, return std::make_pair(std::move(Heading), std::move(SupportedSpellings)); } +static DocumentationData +createDocumentationData(const Record &Documentation, + ArrayRef<const Record *> Attributes) { + DocumentationData Doc(Documentation, Attributes); + StringRef CategoryName = Doc.Category->getValueAsString("Name"); + for (const Record *Attribute : Attributes) { + auto [AttributeHeading, AttributeSpellings] = + GetAttributeHeadingAndSpellings(Documentation, *Attribute, + CategoryName); + if (Doc.Heading.empty()) + Doc.Heading = std::move(AttributeHeading); + else if (Doc.Heading != AttributeHeading) + Doc.Heading += ", " + AttributeHeading; + Doc.SupportedSpellings.merge(AttributeSpellings); + } + return Doc; +} + static void WriteDocumentation(const RecordKeeper &Records, const DocumentationData &Doc, raw_ostream &OS) { if (StringRef Label = Doc.Documentation->getValueAsString("Label"); @@ -5602,7 +5630,7 @@ static void WriteDocumentation(const RecordKeeper &Records, OS << " - "; if (getPragmaAttributeSupport(Records).isAttributedSupported( - *Doc.Attribute)) + *Doc.Attributes.front())) OS << "Yes"; OS << "\n:::\n\n"; } @@ -5626,6 +5654,37 @@ static void WriteDocumentation(const RecordKeeper &Records, OS << "\n\n\n"; } +using AttributesByDocumentation = + DenseMap<const Record *, std::vector<const Record *>>; +using DocumentationByCategory = + std::map<std::string, AttributesByDocumentation>; + +static DocumentationByCategory +GatherAttributeDocumentation(const RecordKeeper &Records) { + DocumentationByCategory DocsByCategory; + for (const Record *Attribute : Records.getAllDerivedDefinitions("Attr")) { + std::vector<const Record *> Docs = + Attribute->getValueAsListOfDefs("Documentation"); + for (const Record *Doc : Docs) { + StringRef CategoryName = + Doc->getValueAsDef("Category")->getValueAsString("Name"); + if (CategoryName == "InternalOnly") { + if (Docs.size() > 1) + PrintFatalError(Doc->getLoc(), + "Attribute is \"InternalOnly\", but has multiple " + "documentation categories"); + continue; + } + + // Reverse Attr -> Documentation into Category -> Documentation -> Attrs. + // Attribute insertion order determines combined heading and spelling + // order for documentation shared by multiple attributes. + DocsByCategory[CategoryName.str()][Doc].push_back(Attribute); + } + } + return DocsByCategory; +} + void GetListOfUndocumentedAttributes( const RecordKeeper &Records, std::vector<const Record *> &UndocumentedAttrs) { @@ -5678,111 +5737,45 @@ void EmitClangAttrDocs(const RecordKeeper &Records, raw_ostream &OS) { return; } - OS << Documentation->getValueAsString("Intro") << "\n"; - - // Gather the Documentation lists from each of the attributes, based on the - // category provided. - struct CategoryLess { - bool operator()(const Record *L, const Record *R) const { - return L->getValueAsString("Name") < R->getValueAsString("Name"); - } - }; - - std::map<const Record *, std::map<uint32_t, DocumentationData>, CategoryLess> - MergedDocs; - - std::vector<DocumentationData> UndocumentedDocs; - const Record *UndocumentedCategory = nullptr; - - // Collect documentation data, grouping by category and heading. - for (const auto *A : Records.getAllDerivedDefinitions("Attr")) { - const Record &Attr = *A; - std::vector<const Record *> Docs = - Attr.getValueAsListOfDefs("Documentation"); - - for (const auto *D : Docs) { - const Record &Doc = *D; - const Record *Category = Doc.getValueAsDef("Category"); - // If the category is "InternalOnly", then there cannot be any other - // documentation categories (otherwise, the attribute would be - // emitted into the docs). - StringRef Cat = Category->getValueAsString("Name"); - if (Cat == "InternalOnly" && Docs.size() > 1) - PrintFatalError(Doc.getLoc(), - "Attribute is \"InternalOnly\", but has multiple " - "documentation categories"); - - if (Cat == "InternalOnly") - continue; - - // Track the Undocumented category Record for later grouping - if (Cat == "Undocumented" && !UndocumentedCategory) - UndocumentedCategory = Category; - - // Generate Heading and Spellings. - auto HeadingAndSpellings = - GetAttributeHeadingAndSpellings(Doc, Attr, Cat); - - // Handle Undocumented category separately - no content merging - if (Cat == "Undocumented" && UndocumentedCategory) { - UndocumentedDocs.push_back( - DocumentationData(Doc, Attr, std::move(HeadingAndSpellings))); - continue; - } + StringRef Intro = Documentation->getValueAsString("Intro"); + DocumentationByCategory DocsByCategory = + GatherAttributeDocumentation(Records); + + // Emit the top-level file which includes each category sub-file. + OS << "//--- AttributeReference.md\n"; + OS << Intro.trim() << "\n\n"; + for (const auto &[CatName, _] : DocsByCategory) { + OS << ":::{include} AttributeReference/" + << getAttributeReferenceCategoryFileName(CatName) << "\n"; + OS << ":::\n\n"; + } - auto &CategoryDocs = MergedDocs[Category]; - - std::string key = Doc.getValueAsString("Content").str(); - uint32_t keyHash = llvm::hash_value(key); - - // If the content already exists, merge the documentation. - auto It = CategoryDocs.find(keyHash); - if (It != CategoryDocs.end()) { - // Merge heading - if (It->second.Heading != HeadingAndSpellings.first) - It->second.Heading += ", " + HeadingAndSpellings.first; - // Merge spellings - It->second.SupportedSpellings.merge(HeadingAndSpellings.second); - // Merge content - It->second.Documentation = &Doc; // Update reference + // Reduce and sort each category's documentation immediately before emitting + // its split file. + for (const auto &[CategoryName, AttributesByDoc] : DocsByCategory) { + std::vector<DocumentationData> CategoryDocs; + for (const auto &[Doc, Attributes] : AttributesByDoc) { + // Undocumented is a shared sentinel rather than shared prose, so retain + // one entry per attribute for that category. + if (CategoryName == "Undocumented") { + for (const Record *Attribute : Attributes) + CategoryDocs.push_back( + createDocumentationData(*Doc, ArrayRef(Attribute))); } else { - // Create new entry for unique content - CategoryDocs.emplace(keyHash, - DocumentationData(Doc, Attr, HeadingAndSpellings)); + CategoryDocs.push_back(createDocumentationData(*Doc, Attributes)); } } - } - - std::map<const Record *, std::vector<DocumentationData>, CategoryLess> - SplitDocs; - - for (auto &CategoryPair : MergedDocs) { - - std::vector<DocumentationData> MD; - for (auto &DocPair : CategoryPair.second) - MD.push_back(std::move(DocPair.second)); - - SplitDocs.emplace(CategoryPair.first, MD); - } - - // Append Undocumented category entries - if (!UndocumentedDocs.empty() && UndocumentedCategory) { - SplitDocs.emplace(UndocumentedCategory, UndocumentedDocs); - } - - // Having split the attributes out based on what documentation goes where, - // we can begin to generate sections of documentation. - for (auto &I : SplitDocs) { - WriteCategoryHeader(I.first, OS); - sort(I.second, - [](const DocumentationData &D1, const DocumentationData &D2) { - return D1.Heading < D2.Heading; - }); + // Emit docs alphabetically by heading. + llvm::sort(CategoryDocs, + [](const DocumentationData &L, const DocumentationData &R) { + return L.Heading < R.Heading; + }); - // Walk over each of the attributes in the category and write out their - // documentation. - for (const auto &Doc : I.second) + OS << "//--- AttributeReference/" + << getAttributeReferenceCategoryFileName(CategoryName) << "\n"; + WriteCategoryHeader(CategoryDocs.front().Category, OS); + for (const DocumentationData &Doc : CategoryDocs) WriteDocumentation(Records, Doc, OS); } } _______________________________________________ cfe-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
