mnauw created this revision.
mnauw added a reviewer: sammccall.
mnauw added a project: clang-tools-extra.
Herald added subscribers: usaxena95, kadircet, arphaman, jkorous, MaskRay,
ilya-biryukov, mgorny.
Herald added a project: clang.
Add a standalone executable that can read indexed data and output in requested
format.
Typical use is to read RIFF data to dump to YAML for inspection of indexed
data. The YAML (de)serialization has also been extended to aid in this regard.
Repository:
rG LLVM Github Monorepo
https://reviews.llvm.org/D77385
Files:
clang-tools-extra/clangd/CMakeLists.txt
clang-tools-extra/clangd/index-dump/CMakeLists.txt
clang-tools-extra/clangd/index-dump/IndexDumpMain.cpp
clang-tools-extra/clangd/index/YAMLSerialization.cpp
clang-tools-extra/clangd/test/CMakeLists.txt
Index: clang-tools-extra/clangd/test/CMakeLists.txt
===================================================================
--- clang-tools-extra/clangd/test/CMakeLists.txt
+++ clang-tools-extra/clangd/test/CMakeLists.txt
@@ -13,6 +13,7 @@
ClangdTests
# No tests for these, but we should still make sure they build.
clangd-indexer
+ clangd-index-dump
dexp
)
Index: clang-tools-extra/clangd/index/YAMLSerialization.cpp
===================================================================
--- clang-tools-extra/clangd/index/YAMLSerialization.cpp
+++ clang-tools-extra/clangd/index/YAMLSerialization.cpp
@@ -41,6 +41,8 @@
llvm::Optional<clang::clangd::Symbol> Symbol;
llvm::Optional<RefBundle> Refs;
llvm::Optional<clang::clangd::Relation> Relation;
+ llvm::Optional<clang::clangd::IncludeGraphNode> Source;
+ llvm::Optional<clang::tooling::CompileCommand> Cmd;
};
// A class helps YAML to serialize the 32-bit encoded position (Line&Column),
// as YAMLIO can't directly map bitfields.
@@ -53,6 +55,9 @@
namespace llvm {
namespace yaml {
+using clang::clangd::FileDigest;
+using clang::clangd::IncludeGraph;
+using clang::clangd::IncludeGraphNode;
using clang::clangd::Ref;
using clang::clangd::RefKind;
using clang::clangd::Relation;
@@ -65,6 +70,7 @@
using clang::index::SymbolKind;
using clang::index::SymbolLanguage;
using clang::index::SymbolRole;
+using clang::tooling::CompileCommand;
// Helper to (de)serialize the SymbolID. We serialize it as a hex string.
struct NormalizedSymbolID {
@@ -308,6 +314,76 @@
}
};
+struct NormalizedSourceFlag {
+ NormalizedSourceFlag(IO &) {}
+ NormalizedSourceFlag(IO &, IncludeGraphNode::SourceFlag O) {
+ Flag = static_cast<uint8_t>(O);
+ }
+
+ IncludeGraphNode::SourceFlag denormalize(IO &) {
+ return static_cast<IncludeGraphNode::SourceFlag>(Flag);
+ }
+
+ uint8_t Flag = 0;
+};
+
+struct NormalizedFileDigest {
+ NormalizedFileDigest(IO &) {}
+ NormalizedFileDigest(IO &, const FileDigest &Digest) {
+ HexString = llvm::toHex(Digest);
+ }
+
+ static FileDigest fromRaw(llvm::StringRef Raw) {
+ FileDigest Digest;
+ assert(Raw.size() == sizeof(Digest));
+ memcpy(Digest.data(), Raw.data(), Raw.size());
+ return Digest;
+ }
+
+ static llvm::Expected<FileDigest> fromStr(llvm::StringRef Str) {
+ const int RawSize = sizeof(FileDigest);
+ if (Str.size() != RawSize * 2)
+ return llvm::createStringError(llvm::inconvertibleErrorCode(),
+ "Bad ID length");
+ for (char C : Str)
+ if (!llvm::isHexDigit(C))
+ return llvm::createStringError(llvm::inconvertibleErrorCode(),
+ "Bad hex ID");
+ return fromRaw(llvm::fromHex(Str));
+ }
+
+ FileDigest denormalize(IO &I) {
+ auto Digest = fromStr(HexString);
+ if (!Digest) {
+ I.setError(llvm::toString(Digest.takeError()));
+ return FileDigest();
+ }
+ return *Digest;
+ }
+
+ std::string HexString;
+};
+
+template <> struct MappingTraits<IncludeGraphNode> {
+ static void mapping(IO &IO, IncludeGraphNode &Node) {
+ IO.mapRequired("URI", Node.URI);
+ MappingNormalization<NormalizedSourceFlag, IncludeGraphNode::SourceFlag>
+ NSourceFlag(IO, Node.Flags);
+ IO.mapRequired("Flags", NSourceFlag->Flag);
+ MappingNormalization<NormalizedFileDigest, FileDigest> NDigest(IO,
+ Node.Digest);
+ IO.mapRequired("Digest", NDigest->HexString);
+ IO.mapRequired("DirectIncludes", Node.DirectIncludes);
+ }
+};
+
+template <> struct MappingTraits<CompileCommand> {
+ static void mapping(IO &IO, CompileCommand &Cmd) {
+ IO.mapRequired("Directory", Cmd.Directory);
+ IO.mapRequired("CommandLine", Cmd.CommandLine);
+ }
+};
+
template <> struct MappingTraits<VariantEntry> {
static void mapping(IO &IO, VariantEntry &Variant) {
if (IO.mapTag("!Symbol", Variant.Symbol.hasValue())) {
@@ -322,6 +398,14 @@
if (!IO.outputting())
Variant.Relation.emplace();
MappingTraits<Relation>::mapping(IO, *Variant.Relation);
+ } else if (IO.mapTag("!Source", Variant.Source.hasValue())) {
+ if (!IO.outputting())
+ Variant.Source.emplace();
+ MappingTraits<IncludeGraphNode>::mapping(IO, *Variant.Source);
+ } else if (IO.mapTag("!Cmd", Variant.Cmd.hasValue())) {
+ if (!IO.outputting())
+ Variant.Cmd.emplace();
+ MappingTraits<CompileCommand>::mapping(IO, *Variant.Cmd);
}
}
};
@@ -351,6 +435,18 @@
Entry.Relation = R;
Yout << Entry;
}
+ if (O.Sources) {
+ for (const auto &Source : *O.Sources) {
+ VariantEntry Entry;
+ Entry.Source = Source.getValue();
+ Yout << Entry;
+ }
+ }
+ if (O.Cmd) {
+ VariantEntry Entry;
+ Entry.Cmd = *O.Cmd;
+ Yout << Entry;
+ }
}
llvm::Expected<IndexFileIn> readYAML(llvm::StringRef Data) {
@@ -361,6 +457,8 @@
Arena; // store the underlying data of Position::FileURI.
llvm::UniqueStringSaver Strings(Arena);
llvm::yaml::Input Yin(Data, &Strings);
+ IncludeGraph Sources;
+ llvm::Optional<tooling::CompileCommand> Cmd;
while (Yin.setCurrentDocument()) {
llvm::yaml::EmptyContext Ctx;
VariantEntry Variant;
@@ -375,6 +473,17 @@
Refs.insert(Variant.Refs->first, Ref);
if (Variant.Relation)
Relations.insert(*Variant.Relation);
+ if (Variant.Source) {
+ auto &IGN = Variant.Source.getValue();
+ auto Entry = Sources.try_emplace(IGN.URI).first;
+ Entry->getValue() = std::move(IGN);
+ // Fixup refs to refer to map keys which will live on
+ Entry->getValue().URI = Entry->getKey();
+ for (auto &Include : Entry->getValue().DirectIncludes)
+ Include = Sources.try_emplace(Include).first->getKey();
+ }
+ if (Variant.Cmd)
+ Cmd = *Variant.Cmd;
Yin.nextDocument();
}
@@ -382,6 +491,9 @@
Result.Symbols.emplace(std::move(Symbols).build());
Result.Refs.emplace(std::move(Refs).build());
Result.Relations.emplace(std::move(Relations).build());
+ if (Sources.size())
+ Result.Sources = std::move(Sources);
+ Result.Cmd = std::move(Cmd);
return std::move(Result);
}
Index: clang-tools-extra/clangd/index-dump/IndexDumpMain.cpp
===================================================================
--- /dev/null
+++ clang-tools-extra/clangd/index-dump/IndexDumpMain.cpp
@@ -0,0 +1,69 @@
+//===--- IndexerMain.cpp -----------------------------------------*- C++-*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// clangd-index-dump is a tool to dump stored index data (symbols, xrefs)
+//
+//===----------------------------------------------------------------------===//
+
+#include "index/Serialization.h"
+#include "clang/Tooling/ArgumentsAdjusters.h"
+#include "clang/Tooling/CommonOptionsParser.h"
+#include "clang/Tooling/Tooling.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/Signals.h"
+
+namespace clang {
+namespace clangd {
+namespace {
+
+static llvm::cl::opt<IndexFileFormat>
+ Format("format", llvm::cl::desc("Format of the index to be written"),
+ llvm::cl::values(clEnumValN(IndexFileFormat::YAML, "yaml",
+ "human-readable YAML format"),
+ clEnumValN(IndexFileFormat::RIFF, "binary",
+ "binary RIFF format")),
+ llvm::cl::init(IndexFileFormat::YAML));
+
+static llvm::cl::opt<std::string>
+ InputFile(llvm::cl::Positional, llvm::cl::desc("<input index file>"),
+ llvm::cl::Required);
+
+} // namespace
+} // namespace clangd
+} // namespace clang
+
+int main(int argc, const char **argv) {
+ llvm::sys::PrintStackTraceOnErrorSignal(argv[0]);
+
+ const char *Overview = R"(
+ Dumps an index file to standard output in specified index format.
+ )";
+
+ llvm::cl::ParseCommandLineOptions(argc, argv, Overview, /*Errs=*/nullptr);
+
+ // Read input file
+ using clang::clangd::InputFile;
+ auto Buffer = llvm::MemoryBuffer::getFile(InputFile);
+ if (!Buffer) {
+ llvm::errs() << llvm::formatv("Can't open {0}", InputFile) << "\n";
+ return 1;
+ }
+
+ // Auto-detects input format when parsing
+ auto IndexIn = clang::clangd::readIndexFile(Buffer->get()->getBuffer());
+ if (!IndexIn) {
+ llvm::errs() << llvm::toString(IndexIn.takeError()) << "\n";
+ return 1;
+ }
+
+ // Dump output file
+ clang::clangd::IndexFileOut IndexOut(IndexIn.get());
+ IndexOut.Format = clang::clangd::Format;
+ llvm::outs() << IndexOut;
+ return 0;
+}
Index: clang-tools-extra/clangd/index-dump/CMakeLists.txt
===================================================================
--- /dev/null
+++ clang-tools-extra/clangd/index-dump/CMakeLists.txt
@@ -0,0 +1,14 @@
+include_directories(${CMAKE_CURRENT_SOURCE_DIR}/../)
+
+set(LLVM_LINK_COMPONENTS
+ Support
+ )
+
+add_clang_executable(clangd-index-dump
+ IndexDumpMain.cpp
+ )
+
+target_link_libraries(clangd-index-dump
+ PRIVATE
+ clangDaemon
+)
Index: clang-tools-extra/clangd/CMakeLists.txt
===================================================================
--- clang-tools-extra/clangd/CMakeLists.txt
+++ clang-tools-extra/clangd/CMakeLists.txt
@@ -138,6 +138,7 @@
add_subdirectory(fuzzer)
endif()
add_subdirectory(tool)
+add_subdirectory(index-dump)
add_subdirectory(indexer)
add_subdirectory(index/dex/dexp)
_______________________________________________
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
